Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: batch endpoints for column creation and retrieval #206

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

feat: batch endpoints for column creation and retrieval #206

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f5a6782
feat: batch endpoints for column creation and retrieval
darora Mar 19, 2022
6fd3b24
wip
darora Mar 20, 2022
69515d8
chore: use typebox for column create args type
soedirgo Mar 20, 2022
cafcf99
chore(server): validate batch column create
soedirgo Mar 20, 2022
dc86a6e
fix(lib/columns): error handling
soedirgo Mar 20, 2022
358f683
chore(lib/columns): show error for all invalid cols
soedirgo Mar 21, 2022
fe11fbf
fix: don't assign to request.body
soedirgo Mar 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 121 additions & 52 deletions src/lib/PostgresMetaColumns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,28 @@ import { DEFAULT_SYSTEM_SCHEMAS } from './constants'
import { columnsSql } from './sql'
import { PostgresMetaResult, PostgresColumn } from './types'

interface ColumnCreationRequest {
table_id: number
name: string
type: string
default_value?: any
default_value_format?: 'expression' | 'literal'
is_identity?: boolean
identity_generation?: 'BY DEFAULT' | 'ALWAYS'
is_nullable?: boolean
is_primary_key?: boolean
is_unique?: boolean
comment?: string
check?: string
}

interface ColumnBatchInfoRequest {
ids?: string[]
names?: string[]
table?: string
schema?: string
}

export default class PostgresMetaColumns {
query: (sql: string) => Promise<PostgresMetaResult<any>>
metaTables: PostgresMetaTables
Expand Down Expand Up @@ -57,75 +79,130 @@ export default class PostgresMetaColumns {
schema?: string
}): Promise<PostgresMetaResult<PostgresColumn>> {
if (id) {
const regexp = /^(\d+)\.(\d+)$/
if (!regexp.test(id)) {
return { data: null, error: { message: 'Invalid format for column ID' } }
const { data, error } = await this.batchRetrieve({ ids: [id] })
if (data) {
return { data: data[0], error: null }
} else if (error) {
return { data: null, error: error }
}
}
if (name && table) {
const { data, error } = await this.batchRetrieve({ names: [name], table, schema })
if (data) {
return { data: data[0], error: null }
} else if (error) {
return { data: null, error: error }
}
const matches = id.match(regexp) as RegExpMatchArray
const [tableId, ordinalPos] = matches.slice(1).map(Number)
const sql = `${columnsSql} AND c.oid = ${tableId} AND a.attnum = ${ordinalPos};`
}
return { data: null, error: { message: 'Invalid parameters on column retrieve' } }
}

async batchRetrieve({
ids,
names,
table,
schema = 'public',
}: ColumnBatchInfoRequest): Promise<PostgresMetaResult<PostgresColumn[]>> {
if (ids && ids.length > 0) {
const regexp = /^(\d+)\.(\d+)$/
const filteringClauses = ids
.map((id) => {
if (!regexp.test(id)) {
return { data: null, error: { message: 'Invalid format for column ID' } }
}
const matches = id.match(regexp) as RegExpMatchArray
const [tableId, ordinalPos] = matches.slice(1).map(Number)
return `(c.oid = ${tableId} AND a.attnum = ${ordinalPos})`
})
.join(' OR ')
const sql = `${columnsSql} AND (${filteringClauses});`
const { data, error } = await this.query(sql)
if (error) {
return { data, error }
} else if (data.length === 0) {
return { data: null, error: { message: `Cannot find a column with ID ${id}` } }
} else if (data.length < ids.length) {
return { data: null, error: { message: `Cannot find some of the requested columns.` } }
} else {
return { data: data[0], error }
return { data, error }
}
} else if (name && table) {
const sql = `${columnsSql} AND a.attname = ${literal(name)} AND c.relname = ${literal(
} else if (names && names.length > 0 && table) {
const filteringClauses = names.map((name) => `a.attname = ${literal(name)}`).join(' OR ')
const sql = `${columnsSql} AND (${filteringClauses}) AND c.relname = ${literal(
table
)} AND nc.nspname = ${literal(schema)};`
const { data, error } = await this.query(sql)
if (error) {
return { data, error }
} else if (data.length === 0) {
} else if (data.length < names.length) {
return {
data: null,
error: { message: `Cannot find a column named ${name} in table ${schema}.${table}` },
error: { message: `Cannot find some of the requested columns.` },
}
} else {
return { data: data[0], error }
return { data, error }
}
} else {
return { data: null, error: { message: 'Invalid parameters on column retrieve' } }
}
}

async create({
table_id,
name,
type,
default_value,
default_value_format = 'literal',
is_identity = false,
identity_generation = 'BY DEFAULT',
// Can't pick a value as default since regular columns are nullable by default but PK columns aren't
is_nullable,
is_primary_key = false,
is_unique = false,
comment,
check,
}: {
table_id: number
name: string
type: string
default_value?: any
default_value_format?: 'expression' | 'literal'
is_identity?: boolean
identity_generation?: 'BY DEFAULT' | 'ALWAYS'
is_nullable?: boolean
is_primary_key?: boolean
is_unique?: boolean
comment?: string
check?: string
}): Promise<PostgresMetaResult<PostgresColumn>> {
async create(col: ColumnCreationRequest): Promise<PostgresMetaResult<PostgresColumn>> {
const { data, error } = await this.batchCreate([col])
if (data) {
return { data: data[0], error: null }
} else if (error) {
return { data: null, error: error }
}
return { data: null, error: { message: 'Invalid params' } }
}

async batchCreate(cols: ColumnCreationRequest[]): Promise<PostgresMetaResult<PostgresColumn[]>> {
if (cols.length < 1) {
throw new Error('no columns provided for creation')
}
if ([...new Set(cols.map((col) => col.table_id))].length > 1) {
throw new Error('all columns in a single request must share the same table')
}
const { table_id } = cols[0]
const { data, error } = await this.metaTables.retrieve({ id: table_id })
if (error) {
return { data: null, error }
}
const { name: table, schema } = data!

const sqlStrings = cols.map((col) => this.generateColumnCreationSql(col, schema, table))

const sql = `BEGIN;
${sqlStrings.join('\n')}
COMMIT;
`
{
const { error } = await this.query(sql)
if (error) {
return { data: null, error }
}
}
const names = cols.map((col) => col.name)
return await this.batchRetrieve({ names, table, schema })
}

generateColumnCreationSql(
{
name,
type,
default_value,
default_value_format = 'literal',
is_identity = false,
identity_generation = 'BY DEFAULT',
// Can't pick a value as default since regular columns are nullable by default but PK columns aren't
is_nullable,
is_primary_key = false,
is_unique = false,
comment,
check,
}: ColumnCreationRequest,
schema: string,
table: string
) {
let defaultValueClause = ''
if (is_identity) {
if (default_value !== undefined) {
Expand Down Expand Up @@ -159,22 +236,14 @@ export default class PostgresMetaColumns {
: `COMMENT ON COLUMN ${ident(schema)}.${ident(table)}.${ident(name)} IS ${literal(comment)}`

const sql = `
BEGIN;
ALTER TABLE ${ident(schema)}.${ident(table)} ADD COLUMN ${ident(name)} ${typeIdent(type)}
${defaultValueClause}
${isNullableClause}
${isPrimaryKeyClause}
${isUniqueClause}
${checkSql};
${commentSql};
COMMIT;`
{
const { error } = await this.query(sql)
if (error) {
return { data: null, error }
}
}
return await this.retrieve({ name, table, schema })
${commentSql};`
return sql
}

async update(
Expand Down
38 changes: 38 additions & 0 deletions src/server/routes/columns.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ export default async (fastify: FastifyInstance) => {
return data
})

// deprecated: use GET /batch instead
fastify.get<{
Headers: { pg: string }
Params: {
Expand All @@ -54,6 +55,26 @@ export default async (fastify: FastifyInstance) => {
return data
})

fastify.get<{
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've opted to expose these separately rather than have the existing endpoints handle both cases; it gets annoying with the GET accepting a single ID param (and just gnarly in any case).

We can replace the current endpoint as part of a breaking change in the future.

Headers: { pg: string }
Body: any
}>('/batch', async (request, reply) => {
const connectionString = request.headers.pg
const pgMeta = new PostgresMeta({ ...DEFAULT_POOL_CONFIG, connectionString })
const { data, error } = await pgMeta.columns.batchRetrieve({ ids: request.body })
await pgMeta.end()
if (error) {
request.log.error({ error, request: extractRequestForLogging(request) })
reply.code(400)
if (error.message.startsWith('Cannot find')) reply.code(404)
return { error: error.message }
}

return data
})

// deprecated: use POST /batch instead
// TODO (darora): specifying a schema on the routes would both allow for validation, and enable us to mark methods as deprecated
fastify.post<{
Headers: { pg: string }
Body: any
Expand All @@ -69,7 +90,24 @@ export default async (fastify: FastifyInstance) => {
if (error.message.startsWith('Cannot find')) reply.code(404)
return { error: error.message }
}
return data
})

fastify.post<{
Headers: { pg: string }
Body: any
}>('/batch', async (request, reply) => {
const connectionString = request.headers.pg

const pgMeta = new PostgresMeta({ ...DEFAULT_POOL_CONFIG, connectionString })
const { data, error } = await pgMeta.columns.batchCreate(request.body)
await pgMeta.end()
if (error) {
request.log.error({ error, request: extractRequestForLogging(request) })
reply.code(400)
if (error.message.startsWith('Cannot find')) reply.code(404)
return { error: error.message }
}
return data
})

Expand Down
Loading