globalize cache, add column_stats

README.md

@@ -19,7 +19,7 @@ If you don't have git, you can donwload [a zip file](https://github.com/anneb/pg
      npm install
      cp config/dbconfig.example.json config/dbconfig.json
      # now edit config/dbconfig.json for your PostGis database
-     npm start
+     node pgserver.js
      # point your browser to localost:8090 for more info
 
 For interactive data browsing, preview, administration and api documentation, head to [http://localhost:8090](http://localhost:8090).

column_stats.js

+const sqlTableName = require('./utils/sqltablename.js');
+
+const sql = (params, query) => {
+    return `
+      select count(1)::integer as "count", ${params.column} as "value"
+        from ${params.table} 
+          where ${query.geom_column} is not null 
+            group by ${params.column} order by count(1) desc limit 2000;
+    `
+  } // TODO, use sql place holders $1, $2 etc. instead of inserting user-parameters into query
+
+module.exports = function(app, pool, cache) {
+
+  let cacheMiddleWare = async(req, res, next) => {
+    if (!cache) {
+      next();
+      return;
+    }
+    const cacheDir = `${req.params.table}/attrstats/`;
+    const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.params.column?','+req.params.column:''))
+      .replace(/[\W]+/g, '_');
+  
+    const stats = await cache.getCachedFile(cacheDir, key);
+    if (stats) {
+      console.log(`stats cache hit for ${cacheDir}?${key}`);
+      if (stats.length === 0) {
+        res.status(204)
+      }
+      res.header('Content-Type', 'application/json').send(stats);
+      return;
+    } else {
+      res.sendResponse = res.send;
+      res.send = (body) => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          cache.setCachedFile(cacheDir, key, body);
+        }
+        res.sendResponse(body);
+      }
+      next();
+    }
+  }
+  
+ /**
+ * @swagger
+ *
+ * /data/{table}/colstats/{column}:
+ *   get:
+ *     description: get statistics for column
+ *     tags: ['meta']
+ *     produces:
+ *       - application/json
+ *     parameters:
+ *       - name: table
+ *         description: name of postgis table
+ *         in: path
+ *         required: true
+ *         type: string
+ *       - name: column
+ *         description: name of column
+ *         in: path
+ *         type: string
+ *         required: true
+ *       - name: geom_column
+ *         description: name of geometry column (default 'geom')
+ *         in: query
+ *         required: false
+ *     responses:
+ *       200:
+ *         description: json statistics
+ *         content:
+ *            application/json
+ *         schema:
+ *              type: object
+ *              properties:
+ *                table:
+ *                   type: string
+ *                   description: name of table
+ *                column:
+ *                  type: string
+ *                  description: name of column
+ *                numvalues:
+ *                  description: number of different values, null means unknown (>2000)
+ *                  type: integer
+ *                uniquevalues:
+ *                  description: whether or not all values are unique
+ *                  type: boolean
+ *                values:
+ *                  description: array of values sorted by highest count first
+ *                  type: array
+ *                  maxItems: 2000
+ *                  items:
+ *                    type: object
+ *                    properties:
+ *                      value:
+ *                        description: encountered value for column (any type)
+ *                        type: string
+ *                      count: 
+ *                        description: number of geometries with this value
+ *                        type: integer
+ *       204:
+ *         description: no data 
+ *       422:
+ *         description: invalid table or column
+ *       500:
+ *         description: unexpected error
+ */
+    app.get('/data/:table/colstats/:column', cacheMiddleWare, async (req, res)=>{
+        if (!req.query.geom_column) {
+            req.query.geom_column = 'geom'; // default
+        }
+        const sqlString = sql(req.params, req.query);
+        //console.log(sqlString);
+        try {
+            const result = await pool.query(sqlString);
+            const stats = result.rows
+            if (stats.length === 0) {
+                res.status(204).json({});
+                return;
+            }
+            res.json({
+              table: req.params.table,
+              column: req.params.column,
+              numvalues: stats.length < 2000?stats.length:null,
+              uniquevalues: stats[0].value !== null?stats[0].count === 1:stats.length>1?stats[1].count === 1:false,
+              values: stats
+            })
+        } catch(err) {
+            console.log(err);
+            let status = 500;
+            switch (err.code) {
+              case '42P01':
+                // table does not exist
+                status = 422;
+                break;
+              case '42703':
+                // column does not exist
+                status = 422;
+                break;
+              default:
+            }
+            res.status(status).json({error:err.message})
+        }
+    })
+}
\ No newline at end of file

mvt.js

 // based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/mvt.js
 const sqlTableName = require('./utils/sqltablename.js');
 
-const DirCache = require('./utils/dircache.js')
-const cache = new DirCache('./cache');
-
 const sm = require('@mapbox/sphericalmercator');
 const merc = new sm({
   size: 256
 })
 
-let cacheMiddleWare = async(req, res, next) => {
-  const cacheDir = `${req.params.datasource}/mvt/${req.params.z}/${req.params.x}/${req.params.y}`;
-  const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.query.columns?','+req.query.columns:''))
-    .replace(/[\W]+/g, '_');
-
-  const mvt = await cache.getCachedFile(cacheDir, key);
-  if (mvt) {
-    console.log(`cache hit for ${cacheDir}?${key}`);
-    if (mvt.length === 0) {
-      res.status(204)
-    }
-    res.header('Content-Type', 'application/x-protobuf').send(mvt);
-    return;
-  } else {
-    res.sendResponse = res.send;
-    res.send = (body) => {
-      if (res.statusCode >= 200 && res.statusCode < 300) {
-        cache.setCachedFile(cacheDir, key, body);
-      }
-      res.sendResponse(body);
-    }
-    next();
+function queryColumnsNotNull(queryColumns) {
+  if (queryColumns) {
+    return ` and (${queryColumns.split(',').map(column=>`${column} is not null`).join(' or ')})`
   } 
+  return ''
 }
 
 const sql = (params, query) => {
@@ -57,6 +36,7 @@ const sql = (params, query) => {
               ST_MakeEnvelope(${bounds.join()}, 3857), 
               srid
             ) && ${query.geom_column}
+            ${queryColumnsNotNull(query.columns)}
           -- Optional Filter
           ${query.filter ? `AND ${query.filter}` : ''}
       ) r
@@ -64,10 +44,37 @@ const sql = (params, query) => {
     `
   } // TODO, use sql place holders $1, $2 etc. instead of inserting user-parameters into query
 
+module.exports = function(app, pool, cache) {
+
+  let cacheMiddleWare = async(req, res, next) => {
+    if (!cache) {
+      next();
+      return;
+    }
+    const cacheDir = `${req.params.datasource}/mvt/${req.params.z}/${req.params.x}/${req.params.y}`;
+    const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.query.columns?','+req.query.columns:''))
+      .replace(/[\W]+/g, '_');
   
-// TODO add flat-cache
+    const mvt = await cache.getCachedFile(cacheDir, key);
+    if (mvt) {
+      console.log(`cache hit for ${cacheDir}?${key}`);
+      if (mvt.length === 0) {
+        res.status(204)
+      }
+      res.header('Content-Type', 'application/x-protobuf').send(mvt);
+      return;
+    } else {
+      res.sendResponse = res.send;
+      res.send = (body) => {
+        if (res.statusCode >= 200 && res.statusCode < 300) {
+          cache.setCachedFile(cacheDir, key, body);
+        }
+        res.sendResponse(body);
+      }
+      next();
+    }
+  }
   
-module.exports = function(app, pool) {
  /**
  * @swagger
  *
@@ -121,7 +128,7 @@ module.exports = function(app, pool) {
         }
         req.params.table = req.params.datasource;
         const sqlString = sql(req.params, req.query);
-        //console.log(sqlString);
+        // console.log(sqlString);
         try {
             const result = await pool.query(sqlString);
             const mvt = result.rows[0].st_asmvt

pgserver.js

@@ -15,18 +15,25 @@ const dbconfig = require('./config/dbconfig.json');
 const readOnlyPool = new Pool(dbconfig);
 readOnlyPool.connect();
 
+const DirCache = require('./utils/dircache.js')
+const cache = new DirCache(`./cache/${dbconfig.database?dbconfig.database:process.env.PGDATABASE?process.env.PGDATABASE:''}`);
+
+
 const swagger = require('./swagger.js')(app);
 const login = require('./login.js')(app);
 const upload = require('./upload.js')(app);
-const mvt = require('./mvt.js')(app, readOnlyPool);
+const mvt = require('./mvt.js')(app, readOnlyPool, cache);
 const geojson = require('./geojson.js')(app, readOnlyPool);
 const geobuf = require('./geobuf.js')(app, readOnlyPool);
-const list_layers = require('./list_layers.js')(app, readOnlyPool);
-const layer_columns = require('./layer_columns.js')(app, readOnlyPool);
-const bbox = require('./bbox.js')(app, readOnlyPool);
+const listLayers = require('./list_layers.js')(app, readOnlyPool);
+const layerColumns = require('./layer_columns.js')(app, readOnlyPool);
+const bbox = require('./bbox.js')(app, readOnlyPool, cache);
 const query = require('./query.js')(app, readOnlyPool);
+const columnStats = require('./column_stats.js')(app, readOnlyPool, cache);
+
+const server = app.listen(pgserverconfig.port);
+server.setTimeout(600000);
 
-app.listen(pgserverconfig.port);
 console.log(`pgserver listening on port ${pgserverconfig.port}`);
 
 module.exports = app;

public/attrinfo.html

@@ -245,7 +245,7 @@
         const geomColumn = urlParams.get('geom_column');
         document.querySelector('#tablename').innerHTML = fullTableName;
         document.querySelector('#columnname').innerHTML = `${attrName} (${attrType})`;
-        document.querySelector('#back').innerHTML = `<a href="tableinfo.html?table=${fullTableName}">Terug naar layer informatie</a>`
+        document.querySelector('#back').innerHTML = `<a href="tableinfo.html?table=${fullTableName}&geom_column=${geomColumn}">Terug naar layer informatie</a>`
         const parts = fullTableName.split('.');
         const tableName = (parts.length > 1) ? parts[1] : parts[0];
         fetch(`data/query/${fullTableName}?columns=count("${attrName}"),count(distinct+"${attrName}")+as+distinct,min("${attrName}"),max("${attrName}")`).then(response=>{

public/tableinfo.html

@@ -33,13 +33,6 @@
                         li.innerHTML = `<a href="./attrinfo.html?table=${fullTableName}&geom_column=${geomcolumn?geomcolumn:''}&column=${item.field_name}&columntype=${item.field_type}&geomtype=${geomType}"><b>${item.field_name}</b></a> (${item.field_type})`
                         list.appendChild(li);
                     }
-                })
-            } else {
-                const li = document.createElement('li');
-                li.innerHTML = `Error getting list, response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
-                list.appendChild(li);
-            }
-        })
                     fetch(`api/bbox/${fullTableName}${geomcolumn?`?geom_column=${geomcolumn}`:''}`).then(response=>{
                         const bbox = document.querySelector('#bbox');
                         if (response.ok) {
@@ -56,6 +49,14 @@
                             bbox.innerHTML = `Error getting bbox, response: response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
                         }
                     })
+                })
+            } else {
+                const li = document.createElement('li');
+                li.innerHTML = `Error getting list, response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
+                list.appendChild(li);
+            }
+        })
+        
     }
     </script>
 </head>

swagger.js

@@ -23,7 +23,17 @@ const swaggerDefinition = {
 
 const swaggerJSDocOptions = {
     swaggerDefinition,
-    apis: ['./login.js', './mvt.js', './list_layers.js', './layer_columns.js', './bbox.js', './geojson.js', './geobuf.js', './query.js']
+    apis: [
+      './login.js', 
+      './mvt.js', 
+      './list_layers.js', 
+      './layer_columns.js', 
+      './bbox.js', 
+      './geojson.js', 
+      './geobuf.js', 
+      './query.js',
+      './column_stats.js'
+    ]
 }
 
 const swaggerSpec = swaggerJSDoc(swaggerJSDocOptions);