nolan/duckdb-test
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit.

Browse source
This commit is contained in:
Nolan Darilek 2025-04-26 20:22:53 -05:00
commit 694f7ff118
10 changed files with 1873 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.envrc Normal file
View file

@ -0,0 +1,6 @@
use flake
export REDIS_URL=redis://127.0.0.1:5000/
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
export S3_ENDPOINT=http://localhost:5100
export S3_BUCKET=waynav

5
.gitignore vendored Normal file
View file

@ -0,0 +1,5 @@
target
.direnv
*.map*
*.db*
log

1607
Cargo.lock generated Normal file
View file

File diff suppressed because it is too large Load diff

7
Cargo.toml Normal file
View file

@ -0,0 +1,7 @@
[package]
name = "duckdb_test"
version = "0.1.0"
edition = "2021"
[dependencies]
duckdb = { version = "1", features = ["bundled", "extensions-full"] }

114
flake.lock generated Normal file
View file

@ -0,0 +1,114 @@
{
"nodes": {
"naersk": {
"inputs": {
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1743800763,
"narHash": "sha256-YFKV+fxEpMgP5VsUcM6Il28lI0NlpM7+oB1XxbBAYCw=",
"owner": "nix-community",
"repo": "naersk",
"rev": "ed0232117731a4c19d3ee93aa0c382a8fe754b01",
"type": "github"
},
"original": {
"owner": "nix-community",
"ref": "master",
"repo": "naersk",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1744096231,
"narHash": "sha256-kUfx3FKU1Etnua3EaKvpeuXs7zoFiAcli1gBwkPvGSs=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "b2b0718004cc9a5bca610326de0a82e6ea75920b",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgsUnstable": {
"locked": {
"lastModified": 1744098102,
"narHash": "sha256-tzCdyIJj9AjysC3OuKA+tMD/kDEDAF9mICPDU7ix0JA=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c8cd81426f45942bb2906d5ed2fe21d2f19d95b7",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1744168086,
"narHash": "sha256-S9M4HddBCxbbX1CKSyDYgZ8NCVyHcbKnBfoUXeRu2jQ=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "60e405b241edb6f0573f3d9f944617fe33ac4a73",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-24.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"naersk": "naersk",
"nixpkgs": "nixpkgs_2",
"nixpkgsUnstable": "nixpkgsUnstable",
"utils": "utils"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

58
flake.nix Normal file
View file

@ -0,0 +1,58 @@
{
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixos-unstable";
utils.url = "github:numtide/flake-utils";
naersk.url = "github:nix-community/naersk/master";
};
outputs =
{
self,
nixpkgs,
nixpkgsUnstable,
utils,
naersk,
}:
utils.lib.eachDefaultSystem (
system:
let
pkgs = import nixpkgs { inherit system; };
pkgsUnstable = import nixpkgsUnstable { inherit system; };
naersk' = pkgs.callPackage naersk { };
in
{
devShell =
with pkgs;
mkShell.override { stdenv = pkgs.clangStdenv; } {
nativeBuildInputs = [
cargo
rustc
rustfmt
rustPackages.clippy
cargo-watch
pkgsUnstable.dioxus-cli
pkgsUnstable.wasm-bindgen-cli
llvmPackages.bintools
pkg-config
pkgsUnstable.overturemaps
duckdb
pre-commit
overmind
redis
minio
];
buildInputs = [
webkitgtk_4_1
gtk3
xdotool.out
openssl
];
shellHook = ''
# pre-commit install
'';
RUST_SRC_PATH = rustPlatform.rustLibSrc;
};
}
);
}

56
src/main.rs Normal file
View file

@ -0,0 +1,56 @@
use duckdb::{params, Connection};
const ROOT: &str = "s3://overturemaps-us-west-2/release/2025-03-19.0";
fn main() -> Result<(), Box<dyn std::error::Error>> {
let conn = Connection::open("test.map")?;
conn.execute_batch(
r#"
install httpfs;
load httpfs;
install spatial;
load spatial;
set s3_access_key_id = '';
set s3_secret_access_key = '';
set s3_region='us-west-2';
set preserve_insertion_order = false;
"#,
)?;
let mut stmt = conn.prepare("select * from duckdb_settings()")?;
let mut rows = stmt.query([])?;
while let Some(row) = rows.next()? {
let name: String = row.get(0)?;
let value: String = row.get(1)?;
println!("{}: {}", name, value);
}
let xmin = -97.734375;
let xmax = -97.3828125;
let ymin = 30.234375;
let ymax = 30.41015625;
let types = vec![
"theme=places/type=place",
"theme=buildings/type=building",
"theme=transportation/type=segment",
"theme=transportation/type=connector",
];
for t in &types {
let table = t.split('=');
let table = table.last().unwrap();
let root = format!("{ROOT}/{t}/*");
println!("Making table");
println!(
r#"create table {table} as select * from read_parquet("{root}") where bbox.xmin between {xmin} and {xmax} and bbox.ymin between {ymin} and {ymax};"#
);
let query = format!(
r#"create table {table} as select * from read_parquet(?) where bbox.xmin between ? and ? and bbox.ymin between ? and ?;"#
);
conn.execute(&query, params![root, xmin, xmax, ymin, ymax,])?;
println!("Counting");
conn.query_row(&format!("select count() from {table};"), [], |row| {
let count: usize = row.get(0).unwrap();
println!("{count}");
Ok(())
})?;
}
Ok(())
}

5
test1.sh Executable file
View file

@ -0,0 +1,5 @@
#!/bin/sh
cargo build --release
rm -f test.map*
time cargo run --release

3
test2.sh Executable file
View file

@ -0,0 +1,3 @@
#!/bin/sh
rm -f test.map*
time cat test2.sql |duckdb test.map

12
test2.sql Normal file
View file

@ -0,0 +1,12 @@
install httpfs;
load httpfs;
install spatial;
load spatial;
set s3_access_key_id = '';
set s3_secret_access_key = '';
set s3_region='us-west-2';
set preserve_insertion_order = false;
create table place as select * from read_parquet("s3://overturemaps-us-west-2/release/2025-03-19.0/theme=places/type=place/*") where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
create table building as select * from read_parquet("s3://overturemaps-us-west-2/release/2025-03-19.0/theme=buildings/type=building/*") where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
create table segment as select * from read_parquet('s3://overturemaps-us-west-2/release/2025-03-19.0/theme=transportation/type=segment/*') where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
create table connector as select * from read_parquet('s3://overturemaps-us-west-2/release/2025-03-19.0/theme=transportation/type=connector/*') where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;