databased.databased
1import logging 2import sqlite3 3from typing import Any 4 5from griddle import griddy 6from pathier import Pathier, Pathish 7 8 9def dict_factory(cursor: sqlite3.Cursor, row: tuple) -> dict: 10 fields = [column[0] for column in cursor.description] 11 return {column: value for column, value in zip(fields, row)} 12 13 14class Databased: 15 """SQLite3 wrapper.""" 16 17 def __init__( 18 self, 19 dbpath: Pathish = "db.sqlite3", 20 connection_timeout: float = 10, 21 detect_types: bool = True, 22 enforce_foreign_keys: bool = True, 23 commit_on_close: bool = True, 24 logger_encoding: str = "utf-8", 25 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 26 ): 27 """ """ 28 self.path = dbpath 29 self.connection_timeout = connection_timeout 30 self.connection = None 31 self._logger_init(logger_message_format, logger_encoding) 32 self.detect_types = detect_types 33 self.commit_on_close = commit_on_close 34 self.enforce_foreign_keys = enforce_foreign_keys 35 36 def __enter__(self): 37 self.connect() 38 return self 39 40 def __exit__(self, *args, **kwargs): 41 self.close() 42 43 @property 44 def commit_on_close(self) -> bool: 45 """Should commit database before closing connection when `self.close()` is called.""" 46 return self._commit_on_close 47 48 @commit_on_close.setter 49 def commit_on_close(self, should_commit_on_close: bool): 50 self._commit_on_close = should_commit_on_close 51 52 @property 53 def connected(self) -> bool: 54 """Whether this `Databased` instance is connected to the database file or not.""" 55 return self.connection is not None 56 57 @property 58 def connection_timeout(self) -> float: 59 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 60 return self._connection_timeout 61 62 @connection_timeout.setter 63 def connection_timeout(self, timeout: float): 64 self._connection_timeout = timeout 65 66 @property 67 def detect_types(self) -> bool: 68 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 69 70 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 71 """ 72 return self._detect_types 73 74 @detect_types.setter 75 def detect_types(self, should_detect: bool): 76 self._detect_types = should_detect 77 78 @property 79 def enforce_foreign_keys(self) -> bool: 80 return self._enforce_foreign_keys 81 82 @enforce_foreign_keys.setter 83 def enforce_foreign_keys(self, should_enforce: bool): 84 self._enforce_foreign_keys = should_enforce 85 self._set_foreign_key_enforcement() 86 87 @property 88 def indicies(self) -> list[str]: 89 """List of indicies for this database.""" 90 return [ 91 table["name"] 92 for table in self.query( 93 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 94 ) 95 ] 96 97 @property 98 def name(self) -> str: 99 """The name of this database.""" 100 return self.path.stem 101 102 @property 103 def path(self) -> Pathier: 104 """The path to this database file.""" 105 return self._path 106 107 @path.setter 108 def path(self, new_path: Pathish): 109 """If `new_path` doesn't exist, it will be created (including parent folders).""" 110 self._path = Pathier(new_path) 111 if not self.path.exists(): 112 self.path.touch() 113 114 @property 115 def tables(self) -> list[str]: 116 """List of table names for this database.""" 117 return [ 118 table["name"] 119 for table in self.query( 120 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 121 ) 122 ] 123 124 @property 125 def views(self) -> list[str]: 126 """List of view for this database.""" 127 return [ 128 table["name"] 129 for table in self.query( 130 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 131 ) 132 ] 133 134 def _logger_init(self, message_format: str, encoding: str): 135 """:param: `message_format`: `{` style format string.""" 136 self.logger = logging.getLogger(self.name) 137 if not self.logger.hasHandlers(): 138 handler = logging.FileHandler( 139 str(self.path).replace(".", "") + ".log", encoding=encoding 140 ) 141 handler.setFormatter( 142 logging.Formatter( 143 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 144 ) 145 ) 146 self.logger.addHandler(handler) 147 self.logger.setLevel(logging.INFO) 148 149 def _set_foreign_key_enforcement(self): 150 if self.connection: 151 self.connection.execute( 152 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 153 ) 154 155 def add_column(self, table: str, column_def: str): 156 """Add a column to `table`. 157 158 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 159 160 i.e. 161 >>> db = Databased() 162 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 163 self.query(f"ALTER TABLE {table} ADD {column_def};") 164 165 def close(self): 166 """Disconnect from the database. 167 168 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 169 """ 170 if self.connection: 171 if self.commit_on_close: 172 self.commit() 173 self.connection.close() 174 self.connection = None 175 176 def commit(self): 177 """Commit state of database.""" 178 if self.connection: 179 self.connection.commit() 180 self.logger.info("Committed successfully.") 181 else: 182 raise RuntimeError( 183 "Databased.commit(): Can't commit db with no open connection." 184 ) 185 186 def connect(self): 187 """Connect to the database.""" 188 self.connection = sqlite3.connect( 189 self.path, 190 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 191 if self.detect_types 192 else 0, 193 timeout=self.connection_timeout, 194 ) 195 self._set_foreign_key_enforcement() 196 self.connection.row_factory = dict_factory 197 198 def count( 199 self, 200 table: str, 201 column: str = "*", 202 where: str | None = None, 203 distinct: bool = False, 204 ) -> int: 205 """Return number of matching rows in `table` table. 206 207 Equivalent to: 208 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 209 query = ( 210 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 211 ) 212 if where: 213 query += f" WHERE {where}" 214 query += ";" 215 return int(list(self.query(query)[0].values())[0]) 216 217 def create_table(self, table: str, *column_defs: str): 218 """Create a table if it doesn't exist. 219 220 #### :params: 221 222 `table`: Name of the table to create. 223 224 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 225 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 226 columns = ", ".join(column_defs) 227 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 228 self.logger.info(f"'{table}' table created.") 229 230 def delete(self, table: str, where: str | None = None) -> int: 231 """Delete rows from `table` that satisfy the given `where` clause. 232 233 If `where` is `None`, all rows will be deleted. 234 235 Returns the number of deleted rows. 236 237 e.g. 238 >>> db = Databased() 239 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 240 try: 241 if where: 242 self.query(f"DELETE FROM {table} WHERE {where};") 243 else: 244 self.query(f"DELETE FROM {table};") 245 row_count = self.cursor.rowcount 246 self.logger.info( 247 f"Deleted {row_count} rows from '{table}' where '{where}'." 248 ) 249 return row_count 250 except Exception as e: 251 self.logger.exception( 252 f"Error deleting rows from '{table}' where '{where}'." 253 ) 254 raise e 255 256 def describe(self, table: str) -> list[dict]: 257 """Returns information about `table`.""" 258 return self.query(f"pragma table_info('{table}');") 259 260 def drop_column(self, table: str, column: str): 261 """Drop `column` from `table`.""" 262 self.query(f"ALTER TABLE {table} DROP {column};") 263 264 def drop_table(self, table: str) -> bool: 265 """Drop `table` from the database. 266 267 Returns `True` if successful, `False` if not.""" 268 try: 269 self.query(f"DROP TABLE {table};") 270 self.logger.info(f"Dropped table '{table}'.") 271 return True 272 except Exception as e: 273 print(f"{type(e).__name__}: {e}") 274 self.logger.error(f"Failed to drop table '{table}'.") 275 return False 276 277 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 278 """Execute sql script located at `path`.""" 279 if not self.connected: 280 self.connect() 281 assert self.connection 282 return self.connection.executescript( 283 Pathier(path).read_text(encoding) 284 ).fetchall() 285 286 def get_columns(self, table: str) -> tuple[str, ...]: 287 """Returns a list of column names in `table`.""" 288 return tuple( 289 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 290 ) 291 292 def insert( 293 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 294 ) -> int: 295 """Insert rows of `values` into `columns` of `table`. 296 297 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 298 """ 299 max_row_count = 900 300 column_list = "(" + ", ".join(columns) + ")" 301 row_count = 0 302 for i in range(0, len(values), max_row_count): 303 chunk = values[i : i + max_row_count] 304 placeholder = ( 305 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 306 ) 307 logger_values = "\n".join( 308 ( 309 "'(" + ", ".join((str(value) for value in row)) + ")'" 310 for row in chunk 311 ) 312 ) 313 flattened_values = tuple((value for row in chunk for value in row)) 314 try: 315 self.query( 316 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 317 flattened_values, 318 ) 319 self.logger.info( 320 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 321 ) 322 row_count += self.cursor.rowcount 323 except Exception as e: 324 self.logger.exception( 325 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 326 ) 327 raise e 328 return row_count 329 330 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 331 """Execute an SQL query and return the results. 332 333 Ensures that the database connection is opened before executing the command. 334 335 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 336 """ 337 if not self.connected: 338 self.connect() 339 assert self.connection 340 self.cursor = self.connection.cursor() 341 self.cursor.execute(query_, parameters) 342 return self.cursor.fetchall() 343 344 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 345 """Rename a column in `table`.""" 346 self.query( 347 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 348 ) 349 350 def rename_table(self, table_to_rename: str, new_table_name: str): 351 """Rename a table.""" 352 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 353 354 def select( 355 self, 356 table: str, 357 columns: list[str] = ["*"], 358 joins: list[str] | None = None, 359 where: str | None = None, 360 group_by: str | None = None, 361 having: str | None = None, 362 order_by: str | None = None, 363 limit: int | str | None = None, 364 ) -> list[dict]: 365 """Return rows for given criteria. 366 367 For complex queries, use the `databased.query()` method. 368 369 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 370 their corresponding key word in their string, but should otherwise be valid SQL. 371 372 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 373 374 >>> Databased().select( 375 "bike_rides", 376 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 377 where="distance > 20", 378 order_by="distance", 379 desc=True, 380 limit=10 381 ) 382 executes the query: 383 >>> SELECT 384 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 385 FROM 386 bike_rides 387 WHERE 388 distance > 20 389 ORDER BY 390 distance DESC 391 Limit 10;""" 392 query = f"SELECT {', '.join(columns)} FROM {table}" 393 if joins: 394 query += f" {' '.join(joins)}" 395 if where: 396 query += f" WHERE {where}" 397 if group_by: 398 query += f" GROUP BY {group_by}" 399 if having: 400 query += f" HAVING {having}" 401 if order_by: 402 query += f" ORDER BY {order_by}" 403 if limit: 404 query += f" LIMIT {limit}" 405 query += ";" 406 rows = self.query(query) 407 return rows 408 409 @staticmethod 410 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 411 """Returns a tabular grid from `data`. 412 413 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 414 """ 415 return griddy(data, "keys", shrink_to_terminal) 416 417 def update( 418 self, table: str, column: str, value: Any, where: str | None = None 419 ) -> int: 420 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 421 422 If `where` is `None` all rows will be updated. 423 424 Returns the number of updated rows. 425 426 e.g. 427 >>> db = Databased() 428 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 429 try: 430 if where: 431 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 432 else: 433 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 434 row_count = self.cursor.rowcount 435 self.logger.info( 436 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 437 ) 438 return row_count 439 except Exception as e: 440 self.logger.exception( 441 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 442 ) 443 raise e 444 445 def vacuum(self) -> int: 446 """Reduce disk size of database after row/table deletion. 447 448 Returns space freed up in bytes.""" 449 size = self.path.size 450 self.query("VACUUM;") 451 return size - self.path.size
15class Databased: 16 """SQLite3 wrapper.""" 17 18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys 36 37 def __enter__(self): 38 self.connect() 39 return self 40 41 def __exit__(self, *args, **kwargs): 42 self.close() 43 44 @property 45 def commit_on_close(self) -> bool: 46 """Should commit database before closing connection when `self.close()` is called.""" 47 return self._commit_on_close 48 49 @commit_on_close.setter 50 def commit_on_close(self, should_commit_on_close: bool): 51 self._commit_on_close = should_commit_on_close 52 53 @property 54 def connected(self) -> bool: 55 """Whether this `Databased` instance is connected to the database file or not.""" 56 return self.connection is not None 57 58 @property 59 def connection_timeout(self) -> float: 60 """Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.""" 61 return self._connection_timeout 62 63 @connection_timeout.setter 64 def connection_timeout(self, timeout: float): 65 self._connection_timeout = timeout 66 67 @property 68 def detect_types(self) -> bool: 69 """Should use `detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES` when establishing a database connection. 70 71 Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened. 72 """ 73 return self._detect_types 74 75 @detect_types.setter 76 def detect_types(self, should_detect: bool): 77 self._detect_types = should_detect 78 79 @property 80 def enforce_foreign_keys(self) -> bool: 81 return self._enforce_foreign_keys 82 83 @enforce_foreign_keys.setter 84 def enforce_foreign_keys(self, should_enforce: bool): 85 self._enforce_foreign_keys = should_enforce 86 self._set_foreign_key_enforcement() 87 88 @property 89 def indicies(self) -> list[str]: 90 """List of indicies for this database.""" 91 return [ 92 table["name"] 93 for table in self.query( 94 "SELECT name FROM sqlite_Schema WHERE type = 'index';" 95 ) 96 ] 97 98 @property 99 def name(self) -> str: 100 """The name of this database.""" 101 return self.path.stem 102 103 @property 104 def path(self) -> Pathier: 105 """The path to this database file.""" 106 return self._path 107 108 @path.setter 109 def path(self, new_path: Pathish): 110 """If `new_path` doesn't exist, it will be created (including parent folders).""" 111 self._path = Pathier(new_path) 112 if not self.path.exists(): 113 self.path.touch() 114 115 @property 116 def tables(self) -> list[str]: 117 """List of table names for this database.""" 118 return [ 119 table["name"] 120 for table in self.query( 121 "SELECT name FROM sqlite_Schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%';" 122 ) 123 ] 124 125 @property 126 def views(self) -> list[str]: 127 """List of view for this database.""" 128 return [ 129 table["name"] 130 for table in self.query( 131 "SELECT name FROM sqlite_Schema WHERE type = 'view' AND name NOT LIKE 'sqlite_%';" 132 ) 133 ] 134 135 def _logger_init(self, message_format: str, encoding: str): 136 """:param: `message_format`: `{` style format string.""" 137 self.logger = logging.getLogger(self.name) 138 if not self.logger.hasHandlers(): 139 handler = logging.FileHandler( 140 str(self.path).replace(".", "") + ".log", encoding=encoding 141 ) 142 handler.setFormatter( 143 logging.Formatter( 144 message_format, style="{", datefmt="%m/%d/%Y %I:%M:%S %p" 145 ) 146 ) 147 self.logger.addHandler(handler) 148 self.logger.setLevel(logging.INFO) 149 150 def _set_foreign_key_enforcement(self): 151 if self.connection: 152 self.connection.execute( 153 f"pragma foreign_keys = {int(self.enforce_foreign_keys)};" 154 ) 155 156 def add_column(self, table: str, column_def: str): 157 """Add a column to `table`. 158 159 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 160 161 i.e. 162 >>> db = Databased() 163 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 164 self.query(f"ALTER TABLE {table} ADD {column_def};") 165 166 def close(self): 167 """Disconnect from the database. 168 169 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 170 """ 171 if self.connection: 172 if self.commit_on_close: 173 self.commit() 174 self.connection.close() 175 self.connection = None 176 177 def commit(self): 178 """Commit state of database.""" 179 if self.connection: 180 self.connection.commit() 181 self.logger.info("Committed successfully.") 182 else: 183 raise RuntimeError( 184 "Databased.commit(): Can't commit db with no open connection." 185 ) 186 187 def connect(self): 188 """Connect to the database.""" 189 self.connection = sqlite3.connect( 190 self.path, 191 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 192 if self.detect_types 193 else 0, 194 timeout=self.connection_timeout, 195 ) 196 self._set_foreign_key_enforcement() 197 self.connection.row_factory = dict_factory 198 199 def count( 200 self, 201 table: str, 202 column: str = "*", 203 where: str | None = None, 204 distinct: bool = False, 205 ) -> int: 206 """Return number of matching rows in `table` table. 207 208 Equivalent to: 209 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 210 query = ( 211 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 212 ) 213 if where: 214 query += f" WHERE {where}" 215 query += ";" 216 return int(list(self.query(query)[0].values())[0]) 217 218 def create_table(self, table: str, *column_defs: str): 219 """Create a table if it doesn't exist. 220 221 #### :params: 222 223 `table`: Name of the table to create. 224 225 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 226 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 227 columns = ", ".join(column_defs) 228 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 229 self.logger.info(f"'{table}' table created.") 230 231 def delete(self, table: str, where: str | None = None) -> int: 232 """Delete rows from `table` that satisfy the given `where` clause. 233 234 If `where` is `None`, all rows will be deleted. 235 236 Returns the number of deleted rows. 237 238 e.g. 239 >>> db = Databased() 240 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 241 try: 242 if where: 243 self.query(f"DELETE FROM {table} WHERE {where};") 244 else: 245 self.query(f"DELETE FROM {table};") 246 row_count = self.cursor.rowcount 247 self.logger.info( 248 f"Deleted {row_count} rows from '{table}' where '{where}'." 249 ) 250 return row_count 251 except Exception as e: 252 self.logger.exception( 253 f"Error deleting rows from '{table}' where '{where}'." 254 ) 255 raise e 256 257 def describe(self, table: str) -> list[dict]: 258 """Returns information about `table`.""" 259 return self.query(f"pragma table_info('{table}');") 260 261 def drop_column(self, table: str, column: str): 262 """Drop `column` from `table`.""" 263 self.query(f"ALTER TABLE {table} DROP {column};") 264 265 def drop_table(self, table: str) -> bool: 266 """Drop `table` from the database. 267 268 Returns `True` if successful, `False` if not.""" 269 try: 270 self.query(f"DROP TABLE {table};") 271 self.logger.info(f"Dropped table '{table}'.") 272 return True 273 except Exception as e: 274 print(f"{type(e).__name__}: {e}") 275 self.logger.error(f"Failed to drop table '{table}'.") 276 return False 277 278 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 279 """Execute sql script located at `path`.""" 280 if not self.connected: 281 self.connect() 282 assert self.connection 283 return self.connection.executescript( 284 Pathier(path).read_text(encoding) 285 ).fetchall() 286 287 def get_columns(self, table: str) -> tuple[str, ...]: 288 """Returns a list of column names in `table`.""" 289 return tuple( 290 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 291 ) 292 293 def insert( 294 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 295 ) -> int: 296 """Insert rows of `values` into `columns` of `table`. 297 298 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 299 """ 300 max_row_count = 900 301 column_list = "(" + ", ".join(columns) + ")" 302 row_count = 0 303 for i in range(0, len(values), max_row_count): 304 chunk = values[i : i + max_row_count] 305 placeholder = ( 306 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 307 ) 308 logger_values = "\n".join( 309 ( 310 "'(" + ", ".join((str(value) for value in row)) + ")'" 311 for row in chunk 312 ) 313 ) 314 flattened_values = tuple((value for row in chunk for value in row)) 315 try: 316 self.query( 317 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 318 flattened_values, 319 ) 320 self.logger.info( 321 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 322 ) 323 row_count += self.cursor.rowcount 324 except Exception as e: 325 self.logger.exception( 326 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 327 ) 328 raise e 329 return row_count 330 331 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 332 """Execute an SQL query and return the results. 333 334 Ensures that the database connection is opened before executing the command. 335 336 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 337 """ 338 if not self.connected: 339 self.connect() 340 assert self.connection 341 self.cursor = self.connection.cursor() 342 self.cursor.execute(query_, parameters) 343 return self.cursor.fetchall() 344 345 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 346 """Rename a column in `table`.""" 347 self.query( 348 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 349 ) 350 351 def rename_table(self, table_to_rename: str, new_table_name: str): 352 """Rename a table.""" 353 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};") 354 355 def select( 356 self, 357 table: str, 358 columns: list[str] = ["*"], 359 joins: list[str] | None = None, 360 where: str | None = None, 361 group_by: str | None = None, 362 having: str | None = None, 363 order_by: str | None = None, 364 limit: int | str | None = None, 365 ) -> list[dict]: 366 """Return rows for given criteria. 367 368 For complex queries, use the `databased.query()` method. 369 370 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 371 their corresponding key word in their string, but should otherwise be valid SQL. 372 373 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 374 375 >>> Databased().select( 376 "bike_rides", 377 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 378 where="distance > 20", 379 order_by="distance", 380 desc=True, 381 limit=10 382 ) 383 executes the query: 384 >>> SELECT 385 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 386 FROM 387 bike_rides 388 WHERE 389 distance > 20 390 ORDER BY 391 distance DESC 392 Limit 10;""" 393 query = f"SELECT {', '.join(columns)} FROM {table}" 394 if joins: 395 query += f" {' '.join(joins)}" 396 if where: 397 query += f" WHERE {where}" 398 if group_by: 399 query += f" GROUP BY {group_by}" 400 if having: 401 query += f" HAVING {having}" 402 if order_by: 403 query += f" ORDER BY {order_by}" 404 if limit: 405 query += f" LIMIT {limit}" 406 query += ";" 407 rows = self.query(query) 408 return rows 409 410 @staticmethod 411 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 412 """Returns a tabular grid from `data`. 413 414 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 415 """ 416 return griddy(data, "keys", shrink_to_terminal) 417 418 def update( 419 self, table: str, column: str, value: Any, where: str | None = None 420 ) -> int: 421 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 422 423 If `where` is `None` all rows will be updated. 424 425 Returns the number of updated rows. 426 427 e.g. 428 >>> db = Databased() 429 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 430 try: 431 if where: 432 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 433 else: 434 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 435 row_count = self.cursor.rowcount 436 self.logger.info( 437 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 438 ) 439 return row_count 440 except Exception as e: 441 self.logger.exception( 442 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 443 ) 444 raise e 445 446 def vacuum(self) -> int: 447 """Reduce disk size of database after row/table deletion. 448 449 Returns space freed up in bytes.""" 450 size = self.path.size 451 self.query("VACUUM;") 452 return size - self.path.size
SQLite3 wrapper.
18 def __init__( 19 self, 20 dbpath: Pathish = "db.sqlite3", 21 connection_timeout: float = 10, 22 detect_types: bool = True, 23 enforce_foreign_keys: bool = True, 24 commit_on_close: bool = True, 25 logger_encoding: str = "utf-8", 26 logger_message_format: str = "{levelname}|-|{asctime}|-|{message}", 27 ): 28 """ """ 29 self.path = dbpath 30 self.connection_timeout = connection_timeout 31 self.connection = None 32 self._logger_init(logger_message_format, logger_encoding) 33 self.detect_types = detect_types 34 self.commit_on_close = commit_on_close 35 self.enforce_foreign_keys = enforce_foreign_keys
If new_path
doesn't exist, it will be created (including parent folders).
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
Should use detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES
when establishing a database connection.
Changes to this property won't take effect until the current connection, if open, is closed and a new connection opened.
156 def add_column(self, table: str, column_def: str): 157 """Add a column to `table`. 158 159 `column_def` should be in the form `{column_name} {type_name} {constraint}`. 160 161 i.e. 162 >>> db = Databased() 163 >>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")""" 164 self.query(f"ALTER TABLE {table} ADD {column_def};")
Add a column to table
.
column_def
should be in the form {column_name} {type_name} {constraint}
.
i.e.
>>> db = Databased()
>>> db.add_column("rides", "num_stops INTEGER NOT NULL DEFAULT 0")
166 def close(self): 167 """Disconnect from the database. 168 169 Does not call `commit()` for you unless the `commit_on_close` property is set to `True`. 170 """ 171 if self.connection: 172 if self.commit_on_close: 173 self.commit() 174 self.connection.close() 175 self.connection = None
Disconnect from the database.
Does not call commit()
for you unless the commit_on_close
property is set to True
.
177 def commit(self): 178 """Commit state of database.""" 179 if self.connection: 180 self.connection.commit() 181 self.logger.info("Committed successfully.") 182 else: 183 raise RuntimeError( 184 "Databased.commit(): Can't commit db with no open connection." 185 )
Commit state of database.
187 def connect(self): 188 """Connect to the database.""" 189 self.connection = sqlite3.connect( 190 self.path, 191 detect_types=sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES 192 if self.detect_types 193 else 0, 194 timeout=self.connection_timeout, 195 ) 196 self._set_foreign_key_enforcement() 197 self.connection.row_factory = dict_factory
Connect to the database.
199 def count( 200 self, 201 table: str, 202 column: str = "*", 203 where: str | None = None, 204 distinct: bool = False, 205 ) -> int: 206 """Return number of matching rows in `table` table. 207 208 Equivalent to: 209 >>> SELECT COUNT({distinct} {column}) FROM {table} {where};""" 210 query = ( 211 f"SELECT COUNT( {('DISTINCT' if distinct else '')} {column}) FROM {table}" 212 ) 213 if where: 214 query += f" WHERE {where}" 215 query += ";" 216 return int(list(self.query(query)[0].values())[0])
Return number of matching rows in table
table.
Equivalent to:
>>> SELECT COUNT({distinct} {column}) FROM {table} {where};
218 def create_table(self, table: str, *column_defs: str): 219 """Create a table if it doesn't exist. 220 221 #### :params: 222 223 `table`: Name of the table to create. 224 225 `column_defs`: Any number of column names and their definitions in proper Sqlite3 sytax. 226 i.e. `"column_name TEXT UNIQUE"` or `"column_name INTEGER PRIMARY KEY"` etc.""" 227 columns = ", ".join(column_defs) 228 result = self.query(f"CREATE TABLE IF NOT EXISTS {table} ({columns});") 229 self.logger.info(f"'{table}' table created.")
Create a table if it doesn't exist.
:params:
table
: Name of the table to create.
column_defs
: Any number of column names and their definitions in proper Sqlite3 sytax.
i.e. "column_name TEXT UNIQUE"
or "column_name INTEGER PRIMARY KEY"
etc.
231 def delete(self, table: str, where: str | None = None) -> int: 232 """Delete rows from `table` that satisfy the given `where` clause. 233 234 If `where` is `None`, all rows will be deleted. 235 236 Returns the number of deleted rows. 237 238 e.g. 239 >>> db = Databased() 240 >>> db.delete("rides", "distance < 5 AND average_speed < 7")""" 241 try: 242 if where: 243 self.query(f"DELETE FROM {table} WHERE {where};") 244 else: 245 self.query(f"DELETE FROM {table};") 246 row_count = self.cursor.rowcount 247 self.logger.info( 248 f"Deleted {row_count} rows from '{table}' where '{where}'." 249 ) 250 return row_count 251 except Exception as e: 252 self.logger.exception( 253 f"Error deleting rows from '{table}' where '{where}'." 254 ) 255 raise e
Delete rows from table
that satisfy the given where
clause.
If where
is None
, all rows will be deleted.
Returns the number of deleted rows.
e.g.
>>> db = Databased()
>>> db.delete("rides", "distance < 5 AND average_speed < 7")
257 def describe(self, table: str) -> list[dict]: 258 """Returns information about `table`.""" 259 return self.query(f"pragma table_info('{table}');")
Returns information about table
.
261 def drop_column(self, table: str, column: str): 262 """Drop `column` from `table`.""" 263 self.query(f"ALTER TABLE {table} DROP {column};")
Drop column
from table
.
265 def drop_table(self, table: str) -> bool: 266 """Drop `table` from the database. 267 268 Returns `True` if successful, `False` if not.""" 269 try: 270 self.query(f"DROP TABLE {table};") 271 self.logger.info(f"Dropped table '{table}'.") 272 return True 273 except Exception as e: 274 print(f"{type(e).__name__}: {e}") 275 self.logger.error(f"Failed to drop table '{table}'.") 276 return False
Drop table
from the database.
Returns True
if successful, False
if not.
278 def execute_script(self, path: Pathish, encoding: str = "utf-8") -> list[dict]: 279 """Execute sql script located at `path`.""" 280 if not self.connected: 281 self.connect() 282 assert self.connection 283 return self.connection.executescript( 284 Pathier(path).read_text(encoding) 285 ).fetchall()
Execute sql script located at path
.
287 def get_columns(self, table: str) -> tuple[str, ...]: 288 """Returns a list of column names in `table`.""" 289 return tuple( 290 (column["name"] for column in self.query(f"pragma table_info('{table}');")) 291 )
Returns a list of column names in table
.
293 def insert( 294 self, table: str, columns: tuple[str, ...], values: list[tuple[Any, ...]] 295 ) -> int: 296 """Insert rows of `values` into `columns` of `table`. 297 298 Each `tuple` in `values` corresponds to an individual row that is to be inserted. 299 """ 300 max_row_count = 900 301 column_list = "(" + ", ".join(columns) + ")" 302 row_count = 0 303 for i in range(0, len(values), max_row_count): 304 chunk = values[i : i + max_row_count] 305 placeholder = ( 306 "(" + "),(".join((", ".join(("?" for _ in row)) for row in chunk)) + ")" 307 ) 308 logger_values = "\n".join( 309 ( 310 "'(" + ", ".join((str(value) for value in row)) + ")'" 311 for row in chunk 312 ) 313 ) 314 flattened_values = tuple((value for row in chunk for value in row)) 315 try: 316 self.query( 317 f"INSERT INTO {table} {column_list} VALUES {placeholder};", 318 flattened_values, 319 ) 320 self.logger.info( 321 f"Inserted into '{column_list}' columns of '{table}' table values \n{logger_values}." 322 ) 323 row_count += self.cursor.rowcount 324 except Exception as e: 325 self.logger.exception( 326 f"Error inserting into '{column_list}' columns of '{table}' table values \n{logger_values}." 327 ) 328 raise e 329 return row_count
Insert rows of values
into columns
of table
.
Each tuple
in values
corresponds to an individual row that is to be inserted.
331 def query(self, query_: str, parameters: tuple[Any, ...] = tuple()) -> list[dict]: 332 """Execute an SQL query and return the results. 333 334 Ensures that the database connection is opened before executing the command. 335 336 The cursor used to execute the query will be available through `self.cursor` until the next time `self.query()` is called. 337 """ 338 if not self.connected: 339 self.connect() 340 assert self.connection 341 self.cursor = self.connection.cursor() 342 self.cursor.execute(query_, parameters) 343 return self.cursor.fetchall()
Execute an SQL query and return the results.
Ensures that the database connection is opened before executing the command.
The cursor used to execute the query will be available through self.cursor
until the next time self.query()
is called.
345 def rename_column(self, table: str, column_to_rename: str, new_column_name: str): 346 """Rename a column in `table`.""" 347 self.query( 348 f"ALTER TABLE {table} RENAME {column_to_rename} TO {new_column_name};" 349 )
Rename a column in table
.
351 def rename_table(self, table_to_rename: str, new_table_name: str): 352 """Rename a table.""" 353 self.query(f"ALTER TABLE {table_to_rename} RENAME TO {new_table_name};")
Rename a table.
355 def select( 356 self, 357 table: str, 358 columns: list[str] = ["*"], 359 joins: list[str] | None = None, 360 where: str | None = None, 361 group_by: str | None = None, 362 having: str | None = None, 363 order_by: str | None = None, 364 limit: int | str | None = None, 365 ) -> list[dict]: 366 """Return rows for given criteria. 367 368 For complex queries, use the `databased.query()` method. 369 370 Parameters `where`, `group_by`, `having`, `order_by`, and `limit` should not have 371 their corresponding key word in their string, but should otherwise be valid SQL. 372 373 `joins` should contain their key word (`INNER JOIN`, `LEFT JOIN`) in addition to the rest of the sub-statement. 374 375 >>> Databased().select( 376 "bike_rides", 377 "id, date, distance, moving_time, AVG(distance/moving_time) as average_speed", 378 where="distance > 20", 379 order_by="distance", 380 desc=True, 381 limit=10 382 ) 383 executes the query: 384 >>> SELECT 385 id, date, distance, moving_time, AVG(distance/moving_time) as average_speed 386 FROM 387 bike_rides 388 WHERE 389 distance > 20 390 ORDER BY 391 distance DESC 392 Limit 10;""" 393 query = f"SELECT {', '.join(columns)} FROM {table}" 394 if joins: 395 query += f" {' '.join(joins)}" 396 if where: 397 query += f" WHERE {where}" 398 if group_by: 399 query += f" GROUP BY {group_by}" 400 if having: 401 query += f" HAVING {having}" 402 if order_by: 403 query += f" ORDER BY {order_by}" 404 if limit: 405 query += f" LIMIT {limit}" 406 query += ";" 407 rows = self.query(query) 408 return rows
Return rows for given criteria.
For complex queries, use the databased.query()
method.
Parameters where
, group_by
, having
, order_by
, and limit
should not have
their corresponding key word in their string, but should otherwise be valid SQL.
joins
should contain their key word (INNER JOIN
, LEFT JOIN
) in addition to the rest of the sub-statement.
>>> Databased().select(
"bike_rides",
"id, date, distance, moving_time, AVG(distance/moving_time) as average_speed",
where="distance > 20",
order_by="distance",
desc=True,
limit=10
)
executes the query:
>>> SELECT
id, date, distance, moving_time, AVG(distance/moving_time) as average_speed
FROM
bike_rides
WHERE
distance > 20
ORDER BY
distance DESC
Limit 10;
410 @staticmethod 411 def to_grid(data: list[dict], shrink_to_terminal: bool = True) -> str: 412 """Returns a tabular grid from `data`. 413 414 If `shrink_to_terminal` is `True`, the column widths of the grid will be reduced to fit within the current terminal. 415 """ 416 return griddy(data, "keys", shrink_to_terminal)
Returns a tabular grid from data
.
If shrink_to_terminal
is True
, the column widths of the grid will be reduced to fit within the current terminal.
418 def update( 419 self, table: str, column: str, value: Any, where: str | None = None 420 ) -> int: 421 """Update `column` of `table` to `value` for rows satisfying the conditions in `where`. 422 423 If `where` is `None` all rows will be updated. 424 425 Returns the number of updated rows. 426 427 e.g. 428 >>> db = Databased() 429 >>> db.update("rides", "elevation", 100, "elevation < 100")""" 430 try: 431 if where: 432 self.query(f"UPDATE {table} SET {column} = ? WHERE {where};", (value,)) 433 else: 434 self.query(f"UPDATE {table} SET {column} = ?;", (value,)) 435 row_count = self.cursor.rowcount 436 self.logger.info( 437 f"Updated {row_count} rows in '{table}' table to '{column}' = '{value}' where '{where}'." 438 ) 439 return row_count 440 except Exception as e: 441 self.logger.exception( 442 f"Failed to update rows in '{table}' table to '{column}' = '{value}' where '{where}'." 443 ) 444 raise e
Update column
of table
to value
for rows satisfying the conditions in where
.
If where
is None
all rows will be updated.
Returns the number of updated rows.
e.g.
>>> db = Databased()
>>> db.update("rides", "elevation", 100, "elevation < 100")
446 def vacuum(self) -> int: 447 """Reduce disk size of database after row/table deletion. 448 449 Returns space freed up in bytes.""" 450 size = self.path.size 451 self.query("VACUUM;") 452 return size - self.path.size
Reduce disk size of database after row/table deletion.
Returns space freed up in bytes.