Datasets

1-- DuckDB 1.5 or newer. Your Pinax Project ID is the S3 access key,
2-- your Pinax API key the secret.
3INSTALL delta; LOAD delta;
4CREATE SECRET pinax (
5  TYPE s3,
6  KEY_ID '<your Pinax Project ID>',
7  SECRET '<your Pinax API key>',
8  REGION 'us-east-1',
9  ENDPOINT 'datasets.s3.pinax.network',
10  URL_STYLE 'path'
11);
12 
13-- To keep the key out of SQL and history, export AWS_ACCESS_KEY_ID (Project ID)
14-- and AWS_SECRET_ACCESS_KEY (API key) before starting DuckDB, then:
15-- INSTALL aws; LOAD aws;
16-- CREATE SECRET pinax (TYPE s3, PROVIDER credential_chain, CHAIN 'env',
17--   REGION 'us-east-1', ENDPOINT 'datasets.s3.pinax.network', URL_STYLE 'path');
18 
19-- Read through the Delta log, filtered by date.
20SELECT block_num, block_id, timestamp, num_transactions
21FROM delta_scan('s3://eth/blocks')
22WHERE date = DATE '2020-01-01'
23ORDER BY block_num
24LIMIT 10;

1# export PINAX_PROJECT_ID=<your Pinax Project ID> PINAX_API_KEY=<your Pinax API key>
2import datetime
3import os
4 
5import polars as pl
6 
7storage = {
8    "aws_endpoint_url": "https://datasets.s3.pinax.network",
9    "aws_region": "us-east-1",
10    "aws_access_key_id": os.environ["PINAX_PROJECT_ID"],
11    "aws_secret_access_key": os.environ["PINAX_API_KEY"],
12    "aws_virtual_hosted_style_request": "false",
13}
14 
15# Read through the Delta log, filtered by date.
16blocks = (
17    pl.scan_delta("s3://eth/blocks", storage_options=storage)
18    .filter(pl.col("date") == datetime.date(2020, 1, 1))
19    .select("block_num", "block_id", "timestamp", "num_transactions")
20    .head(10)
21    .collect()
22)
23print(blocks)
24 
25# boto3: list under a table (with its trailing slash) or fetch one file.
26import boto3
27from botocore.config import Config
28 
29s3 = boto3.client(
30    "s3",
31    endpoint_url="https://datasets.s3.pinax.network",
32    region_name="us-east-1",
33    aws_access_key_id=os.environ["PINAX_PROJECT_ID"],
34    aws_secret_access_key=os.environ["PINAX_API_KEY"],
35    config=Config(
36        s3={"addressing_style": "path"},
37        request_checksum_calculation="when_required",
38        response_checksum_validation="when_required",
39    ),
40)
41listing = s3.list_objects_v2(Bucket="eth", Prefix="blocks/_delta_log/", MaxKeys=5)
42print([o["Key"] for o in listing.get("Contents", [])])

1# A dedicated profile: Access Key ID = your Pinax Project ID,
2# Secret Access Key = your Pinax API key, region us-east-1.
3aws configure --profile pinax
4aws configure set profile.pinax.s3.addressing_style path
5export AWS_PROFILE=pinax
6PINAX_S3=https://datasets.s3.pinax.network
7 
8# List a table, with its trailing slash (a bucket's root can't be listed)
9aws s3 ls s3://eth/blocks/ --endpoint-url $PINAX_S3
10aws s3 ls s3://eth/blocks/_delta_log/ --endpoint-url $PINAX_S3 | tail -5
11 
12# Download one file.
13aws s3 cp s3://eth/blocks/_delta_log/_last_checkpoint . --endpoint-url $PINAX_S3
14 
15# A date folder can also hold files that compaction replaced: read tables
16# through their Delta log (DuckDB delta_scan, Polars scan_delta), not with aws s3 sync.

1// Node.js only: never put your API key in a web page (browsers can't call the endpoint anyway).
2import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
3 
4const s3 = new S3Client({
5  endpoint: "https://datasets.s3.pinax.network",
6  region: "us-east-1",
7  forcePathStyle: true,
8  credentials: {
9    accessKeyId: process.env.PINAX_PROJECT_ID, // your Pinax Project ID
10    secretAccessKey: process.env.PINAX_API_KEY, // your Pinax API key
11  },
12  requestChecksumCalculation: "WHEN_REQUIRED",
13  responseChecksumValidation: "WHEN_REQUIRED",
14});
15 
16// List under a table, with its trailing slash (a bucket's root can't be listed)
17const { Contents = [] } = await s3.send(
18  new ListObjectsV2Command({ Bucket: "eth", Prefix: "blocks/_delta_log/", MaxKeys: 5 }),
19);
20console.log(Contents.map((object) => object.Key));