mvt cache, previewer, uppercase tablename support

.gitignore

 node_modules
 config/dbconfig.json
-public/files
\ No newline at end of file
+public/files
+cache
\ No newline at end of file

bbox.js

 // based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/bbox.js
 
-// route query
+const sqlTableName = require('./utils/sqltablename.js');
+
+
 const sql = (params, query) => {
     return `
     SELECT 
       ST_Extent(ST_Transform(${query.geom_column}, ${query.srid})) as bbox
   
     FROM 
-      ${params.table}
+      ${sqlTableName(params.table)}
   
     -- Optional where filter
     ${query.filter ? `WHERE ${query.filter}` : '' }

geobuf.js

 // based on https://github.com/tobinbradley/dirt-simple-postgis-http-api/blob/master/routes/geobuf.js
 
-// route query
+const sqlTableName = require('./utils/sqltablename.js');
+
 const sql = (params, query) => {
     let bounds = query.bounds ? query.bounds.split(',').map(Number) : null
     bounds && bounds.length === 3
@@ -20,11 +21,11 @@ const sql = (params, query) => {
         ${query.columns ? `, ${query.columns}` : ''}
   
       FROM
-        ${params.table}
+        ${sqlTableName(params.table)}
         ${
           bounds
             ? `, (SELECT ST_SRID(${query.geom_column}) as srid FROM ${
-                params.table
+                sqlTableName(params.table)
               } LIMIT 1) sq`
             : ''
         }

geojson.js

 // based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/geojson.js
 
-// route query
+const sqlTableName = require('./utils/sqltablename.js');
+
 const sql = (params, query) => {
   let bounds = query.bounds ? query.bounds.split(',').map(Number) : null;
   bounds && bounds.length === 3 ? bounds = merc.bbox(bounds[1], bounds[2], bounds[0]) : null;
@@ -30,8 +31,8 @@ const sql = (params, query) => {
       ` : `'{}'::json AS properties`}
                 
     FROM   
-      ${params.table} AS lg
-      ${bounds ? `, (SELECT ST_SRID(${query.geom_column}) as srid FROM ${params.table} LIMIT 1) sq` : ''}
+      ${sqlTableName(params.table)} AS lg
+      ${bounds ? `, (SELECT ST_SRID(${query.geom_column}) as srid FROM ${sqlTableName(params.table)} LIMIT 1) sq` : ''}
       
     
     -- Optional Filter

mvt.js

 // based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/mvt.js
+const sqlTableName = require('./utils/sqltablename.js');
+
+const DirCache = require('./utils/dircache.js')
+const cache = new DirCache('./cache/mvt');
 
 const sm = require('@mapbox/sphericalmercator');
-const fs = require('fs');
 const merc = new sm({
   size: 256
 })
 
-// route query
+let cacheMiddleWare = async(req, res, next) => {
+  const cacheDir = `${req.params.datasource}/${req.params.z}/${req.params.x}/${req.params.y}`;
+  const key = (req.query.geom_column?req.query.geom_column:'geom') + req.query.columns?','+req.query.columns:'';
+
+  const mvt = await cache.getCachedFile(cacheDir, key);
+  if (mvt) {
+    console.log(`cache hit for ${cacheDir}?${key}`);
+    if (mvt.length === 0) {
+      res.status(204)
+    }
+    res.header('Content-Type', 'application/x-protobuf').send(mvt);
+    return;
+  } else {
+    res.sendResponse = res.send;
+    res.send = (body) => {
+      cache.setCachedFile(cacheDir, key, body);
+      res.sendResponse(body);
+    }
+    next();
+  }
+}
+
 const sql = (params, query) => {
     let bounds = merc.bbox(params.x, params.y, params.z, false, '900913')
   
     return `
     SELECT 
       ST_AsMVT(q, '${params.table}', 4096, 'geom')
-    
     FROM (
       SELECT
         ${query.columns ? `${query.columns},` : ''}
@@ -23,29 +46,25 @@ const sql = (params, query) => {
       bounds[2]
     }, ${bounds[3]}))
         ) geom
-  
       FROM (
         SELECT
           ${query.columns ? `${query.columns},` : ''}
           ${query.geom_column},
           srid
         FROM 
-          ${params.table},
+          ${sqlTableName(params.table)},
           (SELECT ST_SRID(${query.geom_column}) AS srid FROM ${
-      params.table
-    } LIMIT 1) a
-          
+      sqlTableName(params.table)
+    } LIMIT 1) a    
         WHERE       
           ST_transform(
             ST_MakeEnvelope(${bounds.join()}, 3857), 
             srid
           ) && 
           ${query.geom_column}
-  
           -- Optional Filter
           ${query.filter ? `AND ${query.filter}` : ''}
       ) r
-  
     ) q
     `
   } // TODO, use sql place holders $1, $2 etc. instead of inserting user-parameters into query
@@ -101,7 +120,7 @@ module.exports = function(app, pool) {
  *       422:
  *         description: invalid datasource or columnname
  */
-    app.get('/data/:datasource/mvt/:z/:x/:y', async (req, res)=>{
+    app.get('/data/:datasource/mvt/:z/:x/:y', cacheMiddleWare, async (req, res)=>{
         if (!req.query.geom_column) {
             req.query.geom_column = 'geom'; // default
         }

pgserver.js

@@ -24,6 +24,7 @@ const geobuf = require('./geobuf.js')(app, readOnlyPool);
 const list_layers = require('./list_layers.js')(app, readOnlyPool);
 const layer_columns = require('./layer_columns.js')(app, readOnlyPool);
 const bbox = require('./bbox.js')(app, readOnlyPool);
+const query = require('./query.js')(app, readOnlyPool);
 
 app.listen(pgserverconfig.port);
 console.log(`pgserver listening on port ${pgserverconfig.port}`);

public/info.html

@@ -67,7 +67,7 @@
         table.innerHTML = '<tr><th>schema</th><th>name</th><th>geom_column</th><th>srid</th><th>geom_type</th><th>dim</th><th>count</th></tr>' +
             layerInfo.map(item=>`<tr>
                 <td>${item.f_table_schema}</td>
-                <td><a href="tableinfo.html?table=${item.f_table_schema}.${item.f_table_name}&geom_column=${item.f_geometry_column}&srid=${item.srid}&geom_type=${item.type}&dimensions=${item.coord_dimension}&estimated_rows=${item.estimated_rows}">${item.f_table_name}</a></td>
+                <td><a href="tableinfo.html?table=${item.f_table_schema}.${item.f_table_name}&geom_column=${item.f_geometry_column}&srid=${item.srid}&geomtype=${item.type}&dimensions=${item.coord_dimension}&estimated_rows=${item.estimated_rows}">${item.f_table_name}</a></td>
                 <td>${item.f_geometry_column}</td>
                 <td>${item.srid}</td>
                 <td>${item.type}</td>

query.js

+const sqlTableName = require('./utils/sqltablename.js');
+
+const sql = (params, query) => {
+    return `
+    SELECT 
+      ${query.columns}
+  
+    FROM 
+    ${sqlTableName(params.table)}
+  
+    -- Optional Filter
+    ${query.filter ? `WHERE ${query.filter}` : '' }
+  
+    -- Optional Group
+    ${query.group ? `GROUP BY ${query.group}` : '' }
+  
+    -- Optional sort
+    ${query.sort ? `ORDER BY ${query.sort}` : '' }
+  
+    -- Optional limit
+    ${query.limit ? `LIMIT ${query.limit}` : '' }
+  
+    `
+  }
+
+module.exports = function(app, pool) {
+/**
+ * @swagger
+ *
+ * /data/query/{table}:
+ *   get:
+ *     description: Query a table or view.
+ *     tags: ['api']
+ *     produces:
+ *       - application/json
+ *     parameters:
+ *       - name: table
+ *         description: The name of the table or view.
+ *         in: path
+ *         required: true
+ *         type: string
+ *       - name: columns
+ *         description: columns to return (default '*')
+ *         in: query
+ *         required: false
+ *       - name: filter
+ *         description: Optional filter parameters for a SQL WHERE statement.
+ *         in: query
+ *         required: false
+ *         type: string
+ *       - name: sort
+ *         description: Optional sort by column(s).
+ *         in: query
+ *         required: false
+ *         type: string
+ *       - name: limit
+ *         description: Optional limit to the number of output features.
+ *         in: query
+ *         required: false
+ *         type: integer
+ *       - name: group
+ *         description: Optional column(s) to group by.
+ *         in: query
+ *         required: false
+ *         type: string
+ *     responses:
+ *       200:
+ *         description: query result
+ *       422:
+ *         description: invalid table or column name
+ */
+  app.get('/data/query/:table', async (req, res)=> {
+    const sqlString = sql(req.params, req.query);
+    try {
+      const result = await pool.query(sqlString);
+      res.json(result.rows);
+    } catch (err) {
+      res.status(422).json({error: err});
+    }
+  })
+}

swagger.js

@@ -23,7 +23,7 @@ const swaggerDefinition = {
 
 const swaggerJSDocOptions = {
     swaggerDefinition,
-    apis: ['./login.js', './mvt.js', './list_layers.js', './layer_columns.js', './bbox.js', './geojson.js', './geobuf.js']
+    apis: ['./login.js', './mvt.js', './list_layers.js', './layer_columns.js', './bbox.js', './geojson.js', './geobuf.js', './query.js']
 }
 
 const swaggerSpec = swaggerJSDoc(swaggerJSDocOptions);

utils/dircache.js

+const fs = require('fs')
+const path = require('path');
+
+
+module.exports = class DirCache {
+  constructor(cacheRoot) {
+    if (!cacheRoot) {
+      this.cacheRoot = path.resolve('./cache/mvt')
+    } else {
+      this.cacheRoot = cacheRoot;
+    }
+  }
+  getCachedFile(dir, key) {
+    const file = path.join(this.cacheRoot, dir, key);
+    return new Promise((resolve, reject)=>{
+      fs.readFile(file, (err, data) => {
+        if (err) {
+          resolve(null);
+        } else {
+          resolve(data)
+        }
+      })
+    })
+  }
+  setCachedFile(dir, key, data) {
+    const dirName = path.join(this.cacheRoot, dir);
+    const filePath = path.join(dirName, key);
+    console.log(`cache dirname: ${dirName}`);
+    console.log(`filePath: ${filePath}`);
+    fs.mkdirSync(dirName, {recursive: true});
+    return new Promise((resolve, reject) => {
+      fs.writeFile(filePath, data, (err) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve();
+        }
+      })
+    })
+  }
+}
+
+function cacheDirName(params) {
+  return `${path.dirname(__dirname)}/cache/mvt/${params.table}/${params.z}/${params.x}/${params.y}`
+}
+  
+  function cacheFileName(query) {
+    if (query.columns) {
+      return query.columns;
+    }
+    return 'noquery';
+  }
+  
+  function getCache(params, query) {
+    const dirname = cacheDirName(params);
+    const filename = cacheFileName(query);
+    
+    console.log(`getCache: ${dirname}`);
+    return fsPromises.readFile(`${dirname}/${filename}`)
+      .then(data=>data)
+      .catch(error=>null);
+  }
+  
+  function setCache(params, query, data) {
+    const dirname = cacheDirName(params);
+    const filename = cacheFileName(query);
+    
+    console.log(`setCache: ${dirname}`);
+    return fsPromises.writeFile(`${dirname}/${filename}`, data)
+      .then(() => {return})
+      .catch(err=>err);
+  }
+  
+  function lockCache(params, query) {
+    const dirname = cacheDirName(params);
+    const filename = cacheFileName(query);
+    fs.mkdirSync(dirname, {recursive: true});
+    return fsPromises.writeFile(`${dirname}/${filename}.lck`, 'lock', {flag: 'wx'})
+      .then(()=>{
+        return true
+      })
+      .catch(err=>{
+        return fsPromises.stat(`${dirname}/${filename}.lck`)
+          .then(st=>{
+            console.log(Date.now() - st.ctimeMs);
+            if (Date.now() - st.ctimeMs > 240000) {
+              return unlockCache(params,query).then(()=>lockCache(params,query));
+            } else {
+              return false;
+            }
+          })
+          .catch(err=>{
+            console.log(err);
+            return false;
+          });
+        });
+  }
+  
+  function unlockCache(params, query){
+    const dirname = cacheDirName(params);
+    const filename = cacheFileName(query);
+    return fsPromises.unlink(`${dirname}/${filename}.lck`)
+      .then(()=>true)
+      .catch(err=>{
+        console.log(`unlockCache: error: ${err}`);
+        return false;
+      })
+  }
+  
+  function wait(ms) {
+    return new Promise((r, j)=>setTimeout(r, ms));
+  }
+  
+  async function waitForCache(params, query) {
+    const dirname = cacheDirName(params);
+    const filename = cacheFileName(query);
+    for (let i = 0; i < 180; i++) {
+      console.log(`waiting for cache.. ${i}`);
+      await wait(1000);
+      data = await getCache(params, query);
+      if (data) {
+        console.log(`cache wait done.. ${i}`)
+        return data;
+      }
+    }
+    console.log(`cache wait failed`);
+    return null;
+  }
+  
\ No newline at end of file

utils/sqltablename.js

+module.exports = function(tableName) {
+    return tableName
+      .split('.')
+      .map(part=>{
+        if (part.search(/^[0-9]|[A-Z]|\s/) === -1) {
+          return part;
+        }
+        let newPart = part;
+        if (part.search('"') !== -1) {
+          newPart = part.replace(/"/g, '\"')
+        }
+        return `"${newPart}"`
+      })
+      .join('.')
+  }
\ No newline at end of file