✨ auto alter table key definition (#103)

Support for modifying schema key definitions without wasting the table content. It has little support for renaming, in fact, renaming should be avoided for the time being.
 
✨ New: 
  - `db:execute` for executing statement without any return.
  - emmylua classes`SqlSchemaKeyDefinition` and `SqliteActions`.
  - when a key has default, then all the columns with nulls will be replaced with the default.
  - support for auto altering key to reference a foreign key.

🐛 Fixes 
  - when a foreign_keys is enabled on a connection, closing and opening disables it.

♻️ Changes
  - rename `db.sqlite_opts` to `db.opts`.

✅ Added Tests
  - auto alter: simple rename with idetnical number of keys
  - auto alter: simple rename with idetnical number of keys with a key turned to be required
  - auto alter: more than one rename with idetnical number of keys
  - auto alter: more than one rename with idetnical number of keys + default without required = true
  - auto alter: transform to foreign key
  - auto alter: pass sqlite.org tests

Author: kkharji

doc/sql.txt

@@ -16,6 +16,21 @@ SQLDatabase                                                      *SQLDatabase*
         {conn} (sqlite3_blob)  sqlite connection c object.
 
 
+SqlSchemaKeyDefinition                                *SqlSchemaKeyDefinition*
+
+
+    Fields: ~
+        {cid}       (number)         column index
+        {name}      (string)         column key
+        {type}      (string)         column type
+        {required}  (boolean)        whether the column key is required or not
+        {primary}   (boolean)        whether the column is a primary key
+        {default}   (string)         the default value of the column
+        {reference} (string)         table_name.column
+        {on_update} (SqliteActions)  what to do when the key gets updated
+        {on_delete} (SqliteActions)  what to do when the key gets deleted
+
+
 SQLQuerySpec                                                    *SQLQuerySpec*
     Query spec that are passed to a number of db: methods.
 
@@ -200,6 +215,17 @@ DB:eval({statement}, {params})                                     *DB:eval()*
         evaluate with named arguments.
 
 
+DB:execute({statement})                                         *DB:execute()*
+    Execute statement without any return
+
+
+    Parameters: ~
+        {statement} (string)  statement to be executed
+
+    Return: ~
+        boolean: true if successful, error out if not.
+
+
 DB:exists({tbl_name})                                            *DB:exists()*
     Check if a table with {tbl_name} exists in sqlite db
 
@@ -221,8 +247,8 @@ DB:create({tbl_name}, {schema})                                  *DB:create()*
 
 
     Parameters: ~
-        {tbl_name} (string)  table name
-        {schema}   (table)   the table keys/column and their types
+        {tbl_name} (string)                                 table name
+        {schema}   (table<string, SqlSchemaKeyDefinition>)
 
     Return: ~
         boolean
@@ -255,7 +281,7 @@ DB:schema({tbl_name})                                            *DB:schema()*
         {tbl_name} (string)  the table name.
 
     Return: ~
-        table list of keys or keys and their type.
+        table<string, SqlSchemaKeyDefinition>
 
 
 DB:insert({tbl_name}, {rows})                                    *DB:insert()*
@@ -375,8 +401,8 @@ tbl:new({db}, {name}, {schema})                                    *tbl:new()*
 
     Parameters: ~
         {db}     (SQLDatabase)
-        {name}   (string)       table name
-        {schema} (table)        table schema
+        {name}   (string)                                 table name
+        {schema} (table<string, SqlSchemaKeyDefinition>)
 
     Return: ~
         SQLTable
@@ -391,7 +417,7 @@ tbl:extend({db}, {name}, {schema})                              *tbl:extend()*
     Parameters: ~
         {db}     (SQLDatabase)
         {name}   (string)
-        {schema} (table)
+        {schema} (table<string, SqlSchemaKeyDefinition>)
 
     Return: ~
         SQLTableExt
@@ -404,10 +430,10 @@ tbl:schema({schema})                                            *tbl:schema()*
 
 
     Parameters: ~
-        {schema} (table)  table schema definition
+        {schema} (table<string, SqlSchemaKeyDefinition>)
 
     Return: ~
-        table table | boolean
+        table<string, SqlSchemaKeyDefinition> | boolean
 
     Usage: ~
         `projects:schema()` get project table schema.

lua/sql.lua

@@ -24,10 +24,28 @@ local flags = clib.flags
 local DB = {}
 DB.__index = DB
 
+---@class SqlSchemaKeyDefinition
+---@field cid number: column index
+---@field name string: column key
+---@field type string: column type
+---@field required boolean: whether the column key is required or not
+---@field primary boolean: whether the column is a primary key
+---@field default string: the default value of the column
+---@field reference string: table_name.column
+---@field on_update SqliteActions: what to do when the key gets updated
+---@field on_delete SqliteActions: what to do when the key gets deleted
+
 ---@class SQLQuerySpec @Query spec that are passed to a number of db: methods.
 ---@field where table: key and value
 ---@field values table: key and value to updated.
 
+---@alias SqliteActions
+---| '"no action"' : when a parent key is modified or deleted from the database, no special action is taken.
+---| '"restrict"' : prohibites from deleting/modifying a parent key when a child key is mapped to it.
+---| '"null"' : when a parent key is deleted/modified, the child key that mapped to the parent key gets set to null.
+---| '"default"' : similar to "null", except that sets to the column's default value instead of NULL.
+---| '"cascade"' : propagates the delete or update operation on the parent key to each dependent child key.
+
 ---Get a table schema, or execute a given function to get it
 ---@param schema table|nil
 ---@param self SQLDatabase
@@ -69,14 +87,14 @@ function DB:open(uri, opts, noconn)
       uri = uri,
       conn = not noconn and clib.connect(uri, opts) or nil,
       closed = noconn and true or false,
-      sqlite_opts = opts,
+      opts = opts or {},
       modified = false,
       created = not noconn and os.date "%Y-%m-%d %H:%M:%S" or nil,
       tbl_schemas = {},
     }, self)
   else
     if self.closed or self.closed == nil then
-      self.conn = clib.connect(self.uri, self.sqlite_opts)
+      self.conn = clib.connect(self.uri, self.opts)
       self.created = os.date "%Y-%m-%d %H:%M:%S"
       self.closed = false
     end
@@ -239,6 +257,14 @@ function DB:eval(statement, params)
   return res
 end
 
+---Execute statement without any return
+---@param statement string: statement to be executed
+---@return boolean: true if successful, error out if not.
+function DB:execute(statement)
+  local succ = clib.exec_stmt(self.conn, statement) == 0
+  return succ and succ or error(clib.last_errmsg(self.conn))
+end
+
 ---Check if a table with {tbl_name} exists in sqlite db
 ---@param tbl_name string: the table name.
 ---@usage `if not db:exists("todo_tbl") then error("...") end`
@@ -251,13 +277,14 @@ end
 ---Create a new sqlite db table with {name} based on {schema}. if {schema.ensure} then
 ---create only when it does not exists. similar to 'create if not exists'.
 ---@param tbl_name string: table name
----@param schema table: the table keys/column and their types
+---@param schema table<string, SqlSchemaKeyDefinition>
 ---@usage `db:create("todos", {id = {"int", "primary", "key"}, title = "text"})` create table with the given schema.
 ---@return boolean
 function DB:create(tbl_name, schema)
   local req = P.create(tbl_name, schema)
   if req:match "reference" then
-    self:eval "pragma foreign_keys = ON"
+    self:execute "pragma foreign_keys = ON"
+    self.opts.foreign_keys = true
   end
   return self:eval(req)
 end
@@ -273,7 +300,7 @@ end
 
 ---Get {name} table schema, if table does not exist then return an empty table.
 ---@param tbl_name string: the table name.
----@return table list of keys or keys and their type.
+---@return table<string, SqlSchemaKeyDefinition>
 function DB:schema(tbl_name)
   local sch = self:eval(("pragma table_info(%s)"):format(tbl_name))
   local schema = {}
@@ -377,7 +404,7 @@ function DB:delete(tbl_name, where)
   a.is_sqltbl(self, tbl_name, "delete")
 
   if not where then
-    return clib.exec_stmt(self.conn, P.delete(tbl_name)) == 0 and true or clib.last_errmsg(self.conn)
+    return self:execute(P.delete(tbl_name))
   end
 
   where = u.is_nested(where) and where or { where }

lua/sql/assert.lua

@@ -12,6 +12,7 @@ local errors = {
   missing_req_key = "(insert) missing a required key: %s",
   missing_db_object = "'%s' db object is not set. please set it with `tbl.set_db(db)` and try again.",
   outdated_schema = "`%s` does not exists in {`%s`}, schema is outdateset `self.db.tbl_schemas[table_name]` or reload",
+  auto_alter_more_less_keys = "schema defined ~= db schema. Please drop `%s` table first or set ensure to false.",
 }
 
 for key, value in pairs(errors) do
@@ -73,4 +74,10 @@ M.should_have_db_object = function(db, name)
   return true
 end
 
+M.auto_alter_should_have_equal_len = function(len_new, len_old, tname)
+  if len_new - len_old ~= 0 then
+    error(errors.auto_alter_more_less_keys:format(tname))
+  end
+end
+
 return M

lua/sql/parser.lua

@@ -341,7 +341,12 @@ local opts_to_str = function(tbl)
       end
     end,
     default = function(v)
-      return "default " .. v
+      local str = "default "
+      if tbl["required"] then
+        return "on conflict replace " .. str .. v
+      else
+        return str .. v
+      end
     end,
     reference = function(v)
       return ("references %s"):format(v:gsub("%.", "(") .. ")")
@@ -386,13 +391,13 @@ end
 ---@param tbl string: table name
 ---@param defs table: keys and type pairs
 ---@return string: the create sql statement.
-M.create = function(tbl, defs)
+M.create = function(tbl, defs, ignore_ensure)
   if not defs then
     return
   end
   local items = {}
 
-  tbl = defs.ensure and "if not exists " .. tbl or tbl
+  tbl = (defs.ensure and not ignore_ensure) and "if not exists " .. tbl or tbl
 
   for k, v in u.opairs(defs) do
     if k ~= "ensure" then
@@ -410,7 +415,7 @@ M.create = function(tbl, defs)
       end
     end
   end
-  return ("create table %s(%s)"):format(tbl, tconcat(items, ", "))
+  return ("CREATE TABLE %s(%s)"):format(tbl, tconcat(items, ", "))
 end
 
 ---Parse table drop statement
@@ -420,7 +425,102 @@ M.drop = function(tbl)
   return "drop table " .. tbl
 end
 
----Preporcess data insert to sql db.
+-- local same_type = function(new, old)
+--   if not new or not old then
+--     return false
+--   end
+
+--   local tnew, told = type(new), type(old)
+
+--   if tnew == told then
+--     if tnew == "string" then
+--       return new == old
+--     elseif tnew == "table" then
+--       if new[1] and old[1] then
+--         return (new[1] == old[1])
+--       elseif new.type and old.type then
+--         return (new.type == old.type)
+--       elseif new.type and old[1] then
+--         return (new.type == old[1])
+--       elseif new[1] and old.type then
+--         return (new[1] == old.type)
+--       end
+--     end
+--   else
+--     if tnew == "table" and told == "string" then
+--       if new.type == old then
+--         return true
+--       elseif new[1] == old then
+--         return true
+--       end
+--     elseif tnew == "string" and told == "table" then
+--       return old.type == new or old[1] == new
+--     end
+--   end
+--   -- return false
+-- end
+
+---Alter a given table, only support changing key definition
+---@param tname string
+---@param new table<string, SqlSchemaKeyDefinition>
+---@param old table<string, SqlSchemaKeyDefinition>
+M.table_alter_key_defs = function(tname, new, old, dry)
+  local tmpname = tname .. "_new"
+  local create = M.create(tmpname, new, true)
+  local drop = M.drop(tname)
+  local move = "INSERT INTO %s(%s) SELECT %s FROM %s"
+  local rename = ("ALTER TABLE %s RENAME TO %s"):format(tmpname, tname)
+  local with_foregin_key = false
+
+  for _, def in pairs(new) do
+    if def.reference then
+      with_foregin_key = true
+    end
+  end
+
+  local stmt = "PRAGMA foreign_keys=off; BEGIN TRANSACTION; %s; COMMIT;"
+  if not with_foregin_key then
+    stmt = stmt .. " PRAGMA foreign_keys=on"
+  end
+
+  local keys = { new = u.okeys(new), old = u.okeys(old) }
+  local idx = { new = {}, old = {} }
+  local len = { new = #keys.new, old = #keys.old }
+  -- local facts = { extra_key = len.new > len.old, drop_key = len.old > len.new }
+
+  a.auto_alter_should_have_equal_len(len.new, len.old, tname)
+
+  for _, varient in ipairs { "new", "old" } do
+    for k, v in pairs(keys[varient]) do
+      idx[varient][v] = k
+    end
+  end
+
+  for i, v in ipairs(keys.new) do
+    if idx.old[v] and idx.old[v] ~= i then
+      local tmp = keys.old[i]
+      keys.old[i] = v
+      keys.old[idx.old[v]] = tmp
+    end
+  end
+
+  local update_null_vals = {}
+  local update_null_stmt = "UPDATE %s SET %s=%s where %s IS NULL"
+  for key, def in pairs(new) do
+    if def.default and not def.required then
+      tinsert(update_null_vals, update_null_stmt:format(tmpname, key, def.default, key))
+    end
+  end
+  update_null_vals = #update_null_vals == 0 and "" or tconcat(update_null_vals, "; ")
+
+  local new_keys, old_keys = tconcat(keys.new, ", "), tconcat(keys.old, ", ")
+  local insert = move:format(tmpname, new_keys, old_keys, tname)
+  stmt = stmt:format(tconcat({ create, insert, update_null_vals, drop, rename }, "; "))
+
+  return not dry and stmt or insert
+end
+
+---Pre-process data insert to sql db.
 ---for now it's mainly used to for parsing lua tables and boolean values.
 ---It throws when a schema key is required and doesn't exists.
 ---@param rows tinserted row.