Skip to content

Commit 29513c7

Browse files
ideal93ideal93
authored
Feat/add filter token transfer job (HemeraProtocol#167)
* Add postgresql service on CI * Fix address holder repeat * Add user_defined_config * Support WETH deposit & withdraw convert to transfers * Migrate FilterTransactionDataJob to BaseJob file(indexer.jobs.filter_transaction_data_job.FilterTransactionDataJob -> indexer.jobs.base_job.FilterTransactionDataJob)
1 parent b435b82 commit 29513c7

25 files changed

+801
-536
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,31 @@
1-
name: Python application unit test
1+
name: Hemera Indexer Continuous Integration
22

33
on: [pull_request]
44

55
jobs:
6-
test:
7-
6+
build:
87
runs-on: ubuntu-latest
9-
8+
services:
9+
postgres:
10+
image: postgres:15
11+
env:
12+
POSTGRES_USER: hemera
13+
POSTGRES_PASSWORD: password
14+
options: >-
15+
--health-cmd pg_isready
16+
--health-interval 10s
17+
--health-timeout 5s
18+
--health-retries 5
19+
ports:
20+
- 5432:5432
21+
env:
22+
ETHEREUM_PUBLIC_NODE_DEBUG_RPC_URL: '${{ secrets.ETHEREUM_PUBLIC_NODE_DEBUG_RPC_URL }}'
23+
ETHEREUM_PUBLIC_NODE_RPC_URL: '${{ secrets.ETHEREUM_PUBLIC_NODE_RPC_URL }}'
24+
LINEA_PUBLIC_NODE_DEBUG_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_DEBUG_RPC_URL }}'
25+
LINEA_PUBLIC_NODE_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_RPC_URL }}'
26+
POSTGRES_USER: hemera
27+
POSTGRES_PASSWORD: password
28+
POSTGRES_URL: postgresql://hemera:password@localhost:5432/hemera
1029
steps:
1130
- uses: actions/checkout@v2
1231
- name: Set up Python
@@ -20,16 +39,20 @@ jobs:
2039
pip install poetry
2140
poetry update
2241
poetry install -v
42+
poetry show
2343
2444
- name: Set PYTHONPATH
2545
run: echo "PYTHONPATH=\$PYTHONPATH:$(pwd)" >> $GITHUB_ENV
2646

2747
- name: Verify PYTHONPATH
2848
run: echo $PYTHONPATH
2949

30-
- name: Unit Test with pytest
31-
env:
32-
LINEA_PUBLIC_NODE_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_RPC_URL }}'
50+
- name: Init database
51+
run: |
52+
export PYTHONPATH=$(pwd)
53+
make init_db
54+
55+
- name: Pipeline Test with pytest
3356
run: |
3457
export PYTHONPATH=$(pwd)
35-
make test indexer
58+
make test indexer

‎.github/workflows/ut.yaml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Python application unit test
2+
3+
on: [pull_request]
4+
5+
jobs:
6+
test:
7+
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
- uses: actions/checkout@v2
12+
- name: Set up Python
13+
uses: actions/setup-python@v3
14+
with:
15+
python-version: '3.9'
16+
17+
- name: Install dependencies
18+
run: |
19+
pip install --upgrade pip
20+
pip install poetry
21+
poetry update
22+
poetry install -v
23+
24+
- name: Set PYTHONPATH
25+
run: echo "PYTHONPATH=\$PYTHONPATH:$(pwd)" >> $GITHUB_ENV
26+
27+
- name: Verify PYTHONPATH
28+
run: echo $PYTHONPATH
29+
30+
- name: Unit Test with pytest
31+
env:
32+
ETHEREUM_PUBLIC_NODE_DEBUG_RPC_URL: '${{ secrets.ETHEREUM_PUBLIC_NODE_DEBUG_RPC_URL }}'
33+
ETHEREUM_PUBLIC_NODE_RPC_URL: '${{ secrets.ETHEREUM_PUBLIC_NODE_RPC_URL }}'
34+
LINEA_PUBLIC_NODE_DEBUG_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_DEBUG_RPC_URL }}'
35+
LINEA_PUBLIC_NODE_RPC_URL: '${{ secrets.LINEA_PUBLIC_NODE_RPC_URL }}'
36+
run: |
37+
export PYTHONPATH=$(pwd)
38+
make test indexer

‎Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ test:
2020
poetry run pytest -vv -m $(filter-out $@,$(MAKECMDGOALS)); \
2121
fi
2222

23+
2324
PRE_COMMIT_INSTALLED := $(shell command -v pre-commit > /dev/null 2>&1 && echo yes || echo no)
2425

2526
format:
@@ -28,4 +29,8 @@ ifeq ($(PRE_COMMIT_INSTALLED),yes)
2829
@pre-commit run --all-files
2930
else
3031
@echo "Please install pre-commit in your local machine(pip install pre-commit or brew install pre-commit)"
31-
endif
32+
endif
33+
34+
init_db:
35+
@echo "Initializing database..."
36+
poetry run python -m hemera.py init_db

‎cli/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
from cli.aggregates import aggregates
66
from cli.api import api
7+
from cli.init_db import init_db
78
from cli.load import load
89
from cli.reorg import reorg
910
from cli.stream import stream
@@ -30,3 +31,4 @@ def cli(ctx):
3031
cli.add_command(api, "api")
3132
cli.add_command(aggregates, "aggregates")
3233
cli.add_command(reorg, "reorg")
34+
cli.add_command(init_db, "init_db")

‎cli/init_db.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import click
2+
3+
from common.services.postgresql_service import PostgreSQLService
4+
5+
6+
@click.command(context_settings=dict(help_option_names=["-h", "--help"]))
7+
@click.option(
8+
"-pg",
9+
"--postgres-url",
10+
type=str,
11+
required=False,
12+
envvar="POSTGRES_URL",
13+
help="The required postgres connection url." "e.g. postgresql+psycopg2://postgres:[email protected]:5432/ethereum",
14+
)
15+
def init_db(postgres_url):
16+
PostgreSQLService(postgres_url, db_version="head", init_schema=True)

‎cli/stream.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -334,20 +334,29 @@ def stream(
334334
logging.warning("No postgres url provided. Exception recorder will not be useful.")
335335

336336
if config_file:
337+
file_based_config = {}
337338
if not os.path.exists(config_file):
338339
raise click.ClickException(f"Config file {config_file} not found")
339340
with open(config_file, "r") as f:
340341
if config_file.endswith(".json"):
341342
import json
342343

343-
config.update(json.load(f))
344+
file_based_config = json.load(f)
344345
elif config_file.endswith(".yaml") or config_file.endswith(".yml"):
345346
import yaml
346347

347-
config.update(yaml.safe_load(f))
348+
file_based_config = yaml.safe_load(f)
348349
else:
349350
raise click.ClickException(f"Config file {config_file} is not supported)")
350351

352+
if file_based_config.get("chain_id") != config["chain_id"]:
353+
raise click.ClickException(
354+
f"Config file {config_file} is not compatible with chain_id {config['chain_id']}"
355+
)
356+
else:
357+
logging.info(f"Loading config from file: {config_file}, chain_id: {config['chain_id']}")
358+
config.update(file_based_config)
359+
351360
if output_types is None:
352361
entity_types = calculate_entity_value(entity_types)
353362
output_types = list(set(generate_output_types(entity_types)))

indexer-config.yaml renamed to config/indexer-config.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
chain_id: 1
12
opensea_job:
23
seaport_1_6:
34
contract_address: "0x0000000000000068f116a894984e2db1123eb395"
@@ -28,3 +29,6 @@ opensea_job:
2829
fee_addresses:
2930
- "0x0000a26b00c1f0df003000390027140000faa719"
3031
- "0x5b3256965e7c3cf26e11fcaf296dfc8807c01073"
32+
33+
export_tokens_and_transfers_job:
34+
weth_address: "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"

‎indexer/controller/scheduler/job_scheduler.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,9 @@
1111
from enumeration.record_level import RecordLevel
1212
from indexer.exporters.console_item_exporter import ConsoleItemExporter
1313
from indexer.jobs import CSVSourceJob
14-
from indexer.jobs.base_job import BaseExportJob, BaseJob, ExtensionJob
14+
from indexer.jobs.base_job import BaseExportJob, BaseJob, ExtensionJob, FilterTransactionDataJob
1515
from indexer.jobs.check_block_consensus_job import CheckBlockConsensusJob
1616
from indexer.jobs.export_blocks_job import ExportBlocksJob
17-
from indexer.jobs.filter_transaction_data_job import FilterTransactionDataJob
1817
from indexer.utils.abi import bytes_to_hex_str
1918
from indexer.utils.exception_recorder import ExceptionRecorder
2019

‎indexer/controller/scheduler/reorg_scheduler.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
from common.models.tokens import Tokens
99
from common.services.postgresql_service import session_scope
1010
from common.utils.module_loading import import_submodules
11+
from indexer.jobs import FilterTransactionDataJob
1112
from indexer.jobs.base_job import BaseExportJob, BaseJob, ExtensionJob
1213
from indexer.jobs.export_blocks_job import ExportBlocksJob
1314
from indexer.jobs.export_reorg_job import ExportReorgJob
14-
from indexer.jobs.filter_transaction_data_job import FilterTransactionDataJob
1515
from indexer.utils.abi import bytes_to_hex_str
1616

1717
import_submodules("indexer.modules")

‎indexer/domain/token_transfer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -309,10 +309,10 @@ def extract_transfer_from_log(log: Log) -> List[TokenTransfer]:
309309

310310
if topic == transfer_event.get_signature():
311311
token_transfers = handle_transfer_event(log)
312-
# elif topic == deposit_event.get_signature():
313-
# token_transfers = handle_deposit_event(log)
314-
# elif topic == withdraw_event.get_signature():
315-
# token_transfers = handle_withdraw_event(log)
312+
elif topic == deposit_event.get_signature():
313+
token_transfers = handle_deposit_event(log)
314+
elif topic == withdraw_event.get_signature():
315+
token_transfers = handle_withdraw_event(log)
316316
elif topic == single_transfer_event.get_signature():
317317
token_transfers = handle_transfer_single_event(log)
318318
elif topic == batch_transfer_event.get_signature():

0 commit comments

Comments
 (0)