Commit efe0c9de authored by Anne Blankert's avatar Anne Blankert

use estimated bbox for very large tables

parent e799d61a
// based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/bbox.js // based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/bbox.js
const sqlTableName = require('./utils/sqltablename.js'); const sqlTableName = require('./utils/sqltablename.js');
const DirCache = require('./utils/dircache.js')
const cache = new DirCache('./cache');
function splitTableName(params) {
const parts = params.table.split('.');
if (parts.length === 1) {
return {
namespace: 'public',
tablename: parts[0]
}
} else {
return {
namespace: parts[0],
tablename: parts[1]
}
}
}
const getSqlEstimateRows = (params) =>{
const ts = splitTableName(params);
return `select ((c.reltuples/case when c.relpages=0 then 1 else c.relpages end) * (pg_relation_size(c.oid) / (current_setting('block_size')::integer)))::integer as estimated_rows
from pg_class c JOIN pg_namespace n on c.relnamespace = n.oid
where n.nspname='${ts.namespace}' and c.relname='${ts.tablename}'
`;
}
const sqlEstimateBbox = (params, query, estimatedRows) => {
const ts = splitTableName(params);
return `
with srid as
(select st_srid(${query.geom_column}) srid
from ${sqlTableName(params.table)}
where ${query.geom_column} is not null limit 1)
,bboxsrid as
(select ST_EstimatedExtent('${ts.namespace}', '${ts.tablename}', '${query.geom_column}') as bboxsrid)
,bboxll as
(select st_extent(st_transform(st_setsrid(st_envelope(bboxsrid), srid),4326)) bboxll
from bboxsrid, srid)
select ${estimatedRows} as allrows, ${estimatedRows} as geomrows, bboxll,srid,bboxsrid
from bboxll,srid,bboxsrid
`;
}
const sql = (params, query) => { const sqlBbox = (params, query) => {
return ` return `
with srid as with srid as
(select st_srid(${query.geom_column}) srid from ${sqlTableName(params.table)} where ${query.geom_column} is not null limit 1) (select st_srid(${query.geom_column}) srid
,bboxll as from ${sqlTableName(params.table)}
(select ST_Extent(ST_Transform(${query.geom_column}, 4326)) as bboxll, count(*) allrows, count(${query.geom_column}) geomrows from ${sqlTableName(params.table)} where ${query.geom_column} is not null limit 1)
,bboxll as
(select ST_Extent(ST_Transform(${query.geom_column}, 4326)) as bboxll, count(*) allrows, count(${query.geom_column}) geomrows
from ${sqlTableName(params.table)}
-- Optional where filter -- Optional where filter
${query.filter ? `WHERE ${query.filter}` : '' } ${query.filter ? `WHERE ${query.filter}` : '' }
) )
,bboxsrid as ,bboxsrid as
(select st_extent(st_transform(st_setsrid(st_envelope(bboxll),4326),srid)) bboxsrid from bboxll,srid) (select st_extent(st_transform(st_setsrid(st_envelope(bboxll),4326),srid)) bboxsrid
select allrows, geomrows, bboxll,srid,bboxsrid from bboxll,srid,bboxsrid from bboxll,srid)
` select allrows, geomrows, bboxll,srid,bboxsrid
from bboxll,srid,bboxsrid
`;
} }
let cacheMiddleware = async(req, res, next) => { module.exports = function(app, pool, cache) {
let cacheMiddleware = async(req, res, next) => {
if (!cache) {
next();
return;
}
const cacheDir = `${req.params.table}/bbox` const cacheDir = `${req.params.table}/bbox`
const key = ((req.query.geom_column?req.query.geom_column:'geom') const key = ((req.query.geom_column?req.query.geom_column:'geom')
+ (req.query.columns?'_'+req.query.columns:'') + (req.query.columns?'_'+req.query.columns:'')
...@@ -42,9 +89,7 @@ let cacheMiddleware = async(req, res, next) => { ...@@ -42,9 +89,7 @@ let cacheMiddleware = async(req, res, next) => {
} }
next(); next();
} }
} }
module.exports = function(app, pool) {
/** /**
* @swagger * @swagger
* *
...@@ -77,6 +122,9 @@ module.exports = function(app, pool) { ...@@ -77,6 +122,9 @@ module.exports = function(app, pool) {
* schema: * schema:
* type: object * type: object
* properties: * properties:
* estimated:
* type: boolean
* description: result is estimated (true) or precise (false)
* allrows: * allrows:
* type: integer * type: integer
* description: number of rows in table * description: number of rows in table
...@@ -118,12 +166,22 @@ module.exports = function(app, pool) { ...@@ -118,12 +166,22 @@ module.exports = function(app, pool) {
if (!req.query.srid) { if (!req.query.srid) {
req.query.srid = 4326 req.query.srid = 4326
} }
const sqlString = sql(req.params, req.query); let sql = getSqlEstimateRows(req.params);
let estimated = false;
try { try {
const result = await pool.query(sqlString); const estimateResult = await pool.query(sql);
const estimatedRows = estimateResult.rows[0].estimated_rows;
if (estimatedRows < 5000000 || req.query.filter) {
sql = sqlBbox(req.params, req.query);
} else {
sql = sqlEstimateBbox(req.params, req.query, estimatedRows);
estimated = true;
}
const result = await pool.query(sql);
if (result.rows.length === 1) { if (result.rows.length === 1) {
const row = result.rows[0]; const row = result.rows[0];
res.json({ res.json({
estimated: estimated,
allrows: Number(row.allrows), allrows: Number(row.allrows),
geomrows: Number(row.geomrows), geomrows: Number(row.geomrows),
bboxll: row.bboxll?row.bboxll.match(/BOX\((.*)\)/)[1].split(',').map(coord=>coord.split(' ').map(c=>parseFloat(c))):null, bboxll: row.bboxll?row.bboxll.match(/BOX\((.*)\)/)[1].split(',').map(coord=>coord.split(' ').map(c=>parseFloat(c))):null,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment