use client side rendering for the homepage.

@types/parquetjs-lite.d.ts

@@ -0,0 +1,7 @@
+declare module 'parquetjs-lite' {
+    export class ParquetReader {
+      static openFile(filePath: string): Promise<ParquetReader>;
+      getCursor(): any;
+      close(): Promise<void>;
+    }
+  }

Dockerfile

@@ -34,6 +34,9 @@ RUN \
     addgroup --system --gid 1001 nodejs; \
     adduser --system --uid 1001 nextjs
 
+# Public files are served by nextjs.
+COPY --from=builder --link /app/public ./public
+
 # Automatically leverage output traces to reduce image size
 COPY --from=builder --link --chown=1001:1001 /app/.next/standalone ./
 COPY --from=builder --link --chown=1001:1001 /app/.next/static ./.next/static

package.json

@@ -9,8 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@duckdb/duckdb-wasm": "^1.29.0",
     "next": "14.2.15",
-    "parquetjs-lite": "^0.8.7",
     "react": "^18",
     "react-dom": "^18"
   },

python/0_download_files.py

@@ -7,8 +7,8 @@ import random
 import argparse
 import yaml
 
-# Create the "tables" folders if they don't exist
-os.makedirs("tables", exist_ok=True)
+# Create the "public" folders if they don't exist
+os.makedirs("public", exist_ok=True)
 
 # URLs of the files to download
 urls = [
@@ -18,7 +18,7 @@ urls = [
 ]
 
 def download_file(url, overwrite=True):
-    filename = os.path.join("tables", url.split("/")[-1].split("?")[0])
+    filename = os.path.join("public", url.split("/")[-1].split("?")[0])
     
     if not overwrite and os.path.exists(filename):
         print(f"File already exists: {filename}. Skipping download.")
@@ -56,7 +56,7 @@ def main(overwrite):
 
     # Process each downloaded Parquet file
     for url in urls:
-        filename = os.path.join("tables", url.split("/")[-1].split("?")[0])
+        filename = os.path.join("public", url.split("/")[-1].split("?")[0])
         table_name = os.path.splitext(os.path.basename(filename))[0]
 
         # Connect to the Parquet file using DuckDB
@@ -86,8 +86,8 @@ def main(overwrite):
             yaml_content = yaml_content.rstrip()  # Remove trailing spaces
             yaml_content += "\n"
 
-        # Save the YAML content to a file in the "tables" folder
-        yaml_file = os.path.join("tables", f"{table_name}.example.yaml")
+        # Save the YAML content to a file in the "public" folder
+        yaml_file = os.path.join("public", f"{table_name}.example.yaml")
         with open(yaml_file, "w") as file:
             file.write(yaml_content)
 

python/1_parents.py

@@ -23,7 +23,7 @@ query = """
         _id, 
         id, 
         extract_base_models(tags) AS base_models
-    FROM parquet_scan('tables/models.parquet')
+    FROM parquet_scan('public/models.parquet')
 """
 
 start_time = time.time()
@@ -32,7 +32,7 @@ start_time = time.time()
 con.execute(f"CREATE VIEW parent_models AS {query}")
 
 # Write the view to a parquet file
-con.execute("COPY parent_models TO 'tables/parents.parquet' (FORMAT 'parquet')")
+con.execute("COPY parent_models TO 'public/parents.parquet' (FORMAT 'parquet')")
 
 end_time = time.time()
 execution_time = end_time - start_time
@@ -55,5 +55,5 @@ result = con.execute("""
     LIMIT 10
 """).fetchall()
 
-with open("tables/parents.example.yaml", "w") as f:
+with open("public/parents.example.yaml", "w") as f:
     yaml.safe_dump(result, f, default_flow_style=False)

python/2_ancestors.py

@@ -22,7 +22,7 @@ total_start_time = time.perf_counter()
 # Load parents.parquet into an in-memory table
 load_parents_query = """
     CREATE TABLE parents_in_memory AS
-    SELECT * FROM parquet_scan('tables/parents.parquet')
+    SELECT * FROM parquet_scan('public/parents.parquet')
 """
 execute_with_timing(load_parents_query, "Loaded parents.parquet into RAM")
 
@@ -115,7 +115,7 @@ final_output_query = """
         FROM ancestor_children ac
         LEFT JOIN direct_children_mapping dcm ON ac.ancestor = dcm.parent
         ORDER BY all_children_count DESC
-    ) TO 'tables/ancestor_children.parquet' (FORMAT 'parquet')
+    ) TO 'public/ancestor_children.parquet' (FORMAT 'parquet')
 """
 con.execute(final_output_query)
 end_time = time.perf_counter()
@@ -131,7 +131,7 @@ sample_query = """
     LIMIT 10
 """
 sample_data = con.execute(sample_query).fetchall()
-with open("tables/ancestor_children.example.yaml", "w") as f:
+with open("public/ancestor_children.example.yaml", "w") as f:
     yaml.safe_dump(sample_data, f, default_flow_style=False)
 end_time = time.perf_counter()
 logging.info(f"Written sample data to YAML file in {end_time - start_time:.6f} seconds.")

src/app/page.tsx

@@ -1,6 +1,7 @@
-import fs from 'fs';
-import path from 'path';
-import { ParquetReader } from 'parquetjs-lite';
+'use client';
+
+import { useState, useEffect } from 'react';
+import * as duckdb from '@duckdb/duckdb-wasm';
 
 type ModelData = {
   ancestor: string;
@@ -10,41 +11,87 @@ type ModelData = {
   direct_children_count: number | null;
 };
 
-export default async function Home() {
-  try {
-    // Read the Parquet file using parquetjs-lite
-    const parquetFilePath = path.join(process.cwd(), 'tables', 'ancestor_children.parquet');
-    const reader = await ParquetReader.openFile(parquetFilePath);
-    const cursor = reader.getCursor();
-
-    // Read all rows and convert to a JavaScript array
-    const data: ModelData[] = [];
-    let row = null;
-    while (row = await cursor.next()) {
-      data.push({
-        ancestor: row.ancestor,
-        direct_children: row.direct_children,
-        all_children: row.all_children,
-        all_children_count: row.all_children_count,
-        direct_children_count: row.direct_children_count,
-      });
-      // console.log(row.all_children.list.length)
+export default function Home() {
+  const [allModels, setAllModels] = useState<ModelData[]>([]);
+  const [currentPage, setCurrentPage] = useState(1);
+  const [pageSize, setPageSize] = useState(100);
+  const [filterText, setFilterText] = useState('');
+
+  useEffect(() => {
+    async function fetchData() {
+      const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+
+      // Select a bundle based on browser checks
+      const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+
+      const worker_url = URL.createObjectURL(
+        new Blob([`importScripts("${bundle.mainWorker!}");`], { type: 'text/javascript' })
+      );
+
+      // Instantiate the asynchronous version of DuckDB-Wasm
+      const worker = new Worker(worker_url);
+      const logger = new duckdb.ConsoleLogger();
+      const db = new duckdb.AsyncDuckDB(logger, worker);
+      await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+
+      // Register the Parquet file using the URL
+      await db.registerFileURL(
+        'ancestor_children.parquet',
+        `${window.location.origin}/ancestor_children.parquet`,
+        duckdb.DuckDBDataProtocol.HTTP,
+        false
+      );
+
+      // Execute the SQL query using the registered Parquet file
+      const query = `
+        SELECT
+          ancestor,
+          direct_children,
+          all_children,
+          CAST(all_children_count AS INTEGER) AS all_children_count,
+          CAST(direct_children_count AS INTEGER) AS direct_children_count
+        FROM 'ancestor_children.parquet'
+      `;
+      const conn = await db.connect();
+      const result = await conn.query(query);
+
+      // Convert the result to a JavaScript array
+      const data: ModelData[] = result.toArray();
+
+      // Close the connection and terminate the worker
+      await conn.close();
+      await db.terminate();
+
+      setAllModels(data);
     }
-    await reader.close();
+    fetchData();
+  }, []);
 
-    // console.log('Data:', data);
+  const filteredModels = allModels.filter((model) =>
+    model.ancestor.toLowerCase().includes(filterText.toLowerCase())
+  );
 
-    // Get the top 10 models with the most all_children
-    const top10Models = data
-      .sort((a, b) => b.all_children.length - a.all_children.length)
-      .slice(0, 10);
+  const totalPages = Math.ceil(filteredModels.length / pageSize);
 
-    // console.log('Top 10 Models:', top10Models);
+  const paginatedModels = filteredModels.slice(
+    (currentPage - 1) * pageSize,
+    currentPage * pageSize
+  );
 
-    return (
-      <main className="container mx-auto py-8 text-gray-900 dark:text-white">
-        <h1 className="text-4xl font-bold mb-4">Top 10 Models with the Most All Children</h1>
-        {top10Models.length > 0 ? (
+  return (
+    <main className="container mx-auto py-8 text-gray-900 dark:text-white">
+      <h1 className="text-4xl font-bold mb-4">All Models</h1>
+      <div className="mb-4">
+        <input
+          type="text"
+          placeholder="Filter by model name"
+          value={filterText}
+          onChange={(e) => setFilterText(e.target.value)}
+          className="px-4 py-2 border border-gray-300 rounded-md"
+        />
+      </div>
+      {paginatedModels.length > 0 ? (
+        <>
           <table className="table-auto border-collapse w-full">
             <thead>
               <tr>
@@ -54,7 +101,7 @@ export default async function Home() {
               </tr>
             </thead>
             <tbody>
-              {top10Models.map((model, index) => (
+              {paginatedModels.map((model, index) => (
                 <tr key={index} className="border-t border-gray-200 dark:border-gray-700">
                   <td className="px-4 py-2">{model.ancestor}</td>
                   <td className="px-4 py-2 text-right">{model.direct_children_count ?? 0}</td>
@@ -63,18 +110,26 @@ export default async function Home() {
               ))}
             </tbody>
           </table>
-        ) : (
-          <p>No data found.</p>
-        )}
-      </main>
-    );
-  } catch (error) {
-    console.error('Error:', error);
-    return (
-      <main className="container mx-auto py-8 text-gray-900 dark:text-white">
-        <h1 className="text-4xl font-bold mb-4">Error</h1>
-        <p>An error occurred while processing the data: {error.message}</p>
-      </main>
-    );
-  }
+          <div className="mt-4">
+            <button
+              onClick={() => setCurrentPage((prev) => Math.max(prev - 1, 1))}
+              disabled={currentPage === 1}
+              className="px-4 py-2 bg-blue-500 text-white rounded-md mr-2"
+            >
+              Previous
+            </button>
+            <button
+              onClick={() => setCurrentPage((prev) => Math.min(prev + 1, totalPages))}
+              disabled={currentPage === totalPages}
+              className="px-4 py-2 bg-blue-500 text-white rounded-md"
+            >
+              Next
+            </button>
+          </div>
+        </>
+      ) : (
+        <p>No data found.</p>
+      )}
+    </main>
+  );
 }