Initial commit.

.envrc

@@ -0,0 +1,6 @@
+use flake
+export REDIS_URL=redis://127.0.0.1:5000/
+export AWS_ACCESS_KEY_ID=minioadmin
+export AWS_SECRET_ACCESS_KEY=minioadmin
+export S3_ENDPOINT=http://localhost:5100
+export S3_BUCKET=waynav

.gitignore

@@ -0,0 +1,5 @@
+target
+.direnv
+*.map*
+*.db*
+log

Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "duckdb_test"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+duckdb = { version = "1", features = ["bundled", "extensions-full"] }

flake.lock

@@ -0,0 +1,114 @@
+{
+  "nodes": {
+    "naersk": {
+      "inputs": {
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1743800763,
+        "narHash": "sha256-YFKV+fxEpMgP5VsUcM6Il28lI0NlpM7+oB1XxbBAYCw=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "ed0232117731a4c19d3ee93aa0c382a8fe754b01",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "master",
+        "repo": "naersk",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1744096231,
+        "narHash": "sha256-kUfx3FKU1Etnua3EaKvpeuXs7zoFiAcli1gBwkPvGSs=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "b2b0718004cc9a5bca610326de0a82e6ea75920b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgsUnstable": {
+      "locked": {
+        "lastModified": 1744098102,
+        "narHash": "sha256-tzCdyIJj9AjysC3OuKA+tMD/kDEDAF9mICPDU7ix0JA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "c8cd81426f45942bb2906d5ed2fe21d2f19d95b7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1744168086,
+        "narHash": "sha256-S9M4HddBCxbbX1CKSyDYgZ8NCVyHcbKnBfoUXeRu2jQ=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "60e405b241edb6f0573f3d9f944617fe33ac4a73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-24.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "naersk": "naersk",
+        "nixpkgs": "nixpkgs_2",
+        "nixpkgsUnstable": "nixpkgsUnstable",
+        "utils": "utils"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,58 @@
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11";
+    nixpkgsUnstable.url = "github:NixOS/nixpkgs/nixos-unstable";
+    utils.url = "github:numtide/flake-utils";
+    naersk.url = "github:nix-community/naersk/master";
+  };
+
+  outputs =
+    {
+      self,
+      nixpkgs,
+      nixpkgsUnstable,
+      utils,
+      naersk,
+    }:
+    utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+        pkgsUnstable = import nixpkgsUnstable { inherit system; };
+        naersk' = pkgs.callPackage naersk { };
+      in
+      {
+        devShell =
+          with pkgs;
+          mkShell.override { stdenv = pkgs.clangStdenv; } {
+            nativeBuildInputs = [
+              cargo
+              rustc
+              rustfmt
+              rustPackages.clippy
+              cargo-watch
+              pkgsUnstable.dioxus-cli
+              pkgsUnstable.wasm-bindgen-cli
+              llvmPackages.bintools
+              pkg-config
+              pkgsUnstable.overturemaps
+              duckdb
+              pre-commit
+              overmind
+              redis
+              minio
+            ];
+            buildInputs = [
+              webkitgtk_4_1
+              gtk3
+              xdotool.out
+              openssl
+            ];
+            shellHook = ''
+              # pre-commit install
+            '';
+            RUST_SRC_PATH = rustPlatform.rustLibSrc;
+          };
+      }
+    );
+}

src/main.rs

@@ -0,0 +1,56 @@
+use duckdb::{params, Connection};
+
+const ROOT: &str = "s3://overturemaps-us-west-2/release/2025-03-19.0";
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let conn = Connection::open("test.map")?;
+    conn.execute_batch(
+        r#"
+        install httpfs;
+        load httpfs;
+        install spatial;
+        load spatial;
+        set s3_access_key_id = '';
+        set s3_secret_access_key = '';
+        set s3_region='us-west-2';
+        set preserve_insertion_order = false;
+        "#,
+    )?;
+    let mut stmt = conn.prepare("select * from duckdb_settings()")?;
+    let mut rows = stmt.query([])?;
+    while let Some(row) = rows.next()? {
+        let name: String = row.get(0)?;
+        let value: String = row.get(1)?;
+        println!("{}: {}", name, value);
+    }
+    let xmin = -97.734375;
+    let xmax = -97.3828125;
+    let ymin = 30.234375;
+    let ymax = 30.41015625;
+    let types = vec![
+        "theme=places/type=place",
+        "theme=buildings/type=building",
+        "theme=transportation/type=segment",
+        "theme=transportation/type=connector",
+    ];
+    for t in &types {
+        let table = t.split('=');
+        let table = table.last().unwrap();
+        let root = format!("{ROOT}/{t}/*");
+        println!("Making table");
+        println!(
+            r#"create table {table} as select * from read_parquet("{root}") where bbox.xmin between {xmin} and {xmax} and bbox.ymin between {ymin} and {ymax};"#
+        );
+        let query = format!(
+            r#"create table {table} as select * from read_parquet(?) where bbox.xmin between ? and ? and bbox.ymin between ? and ?;"#
+        );
+        conn.execute(&query, params![root, xmin, xmax, ymin, ymax,])?;
+        println!("Counting");
+        conn.query_row(&format!("select count() from {table};"), [], |row| {
+            let count: usize = row.get(0).unwrap();
+            println!("{count}");
+            Ok(())
+        })?;
+    }
+    Ok(())
+}

test1.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+cargo build --release
+rm -f test.map*
+time cargo run --release

test2.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+rm -f test.map*
+time cat test2.sql |duckdb test.map

test2.sql

@@ -0,0 +1,12 @@
+install httpfs;
+load httpfs;
+install spatial;
+load spatial;
+set s3_access_key_id = '';
+set s3_secret_access_key = '';
+set s3_region='us-west-2';
+set preserve_insertion_order = false;
+create table place as select * from read_parquet("s3://overturemaps-us-west-2/release/2025-03-19.0/theme=places/type=place/*") where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
+create table building as select * from read_parquet("s3://overturemaps-us-west-2/release/2025-03-19.0/theme=buildings/type=building/*") where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
+create table segment as select * from read_parquet('s3://overturemaps-us-west-2/release/2025-03-19.0/theme=transportation/type=segment/*') where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;
+create table connector as select * from read_parquet('s3://overturemaps-us-west-2/release/2025-03-19.0/theme=transportation/type=connector/*') where bbox.xmin between -97.734375 and -97.3828125 and bbox.ymin between 30.234375 and 30.41015625;