From 366d7865ba5e554e35417ca0947b965bf87bd0d4 Mon Sep 17 00:00:00 2001 From: Wendell Jones Date: Tue, 20 May 2025 09:24:29 -0400 Subject: [PATCH] initial commit --- .gitignore | 13 + .readthedocs.yaml | 13 + .travis.yml | 50 + AUTHORS.rst | 29 + Changelog.rst | 347 +++++ LICENSE | 29 + MANIFEST.in | 4 + META.json | 57 + Makefile | 27 + README.rst | 43 + docs/Makefile | 89 ++ docs/augmentdb.rst | 30 + docs/augobjects.rst | 42 + docs/cast.rst | 39 + docs/cfgobjects.rst | 97 ++ docs/cmdargs.rst | 91 ++ docs/collation.rst | 31 + docs/column.rst | 52 + docs/conf.py | 197 +++ docs/config.rst | 88 ++ docs/configitems.rst | 175 +++ docs/constraint.rst | 109 ++ docs/conversion.rst | 33 + docs/database.rst | 90 ++ docs/dbaugment.rst | 108 ++ docs/dbobject.rst | 101 ++ docs/dbtoyaml.rst | 293 +++++ docs/devel.rst | 97 ++ docs/eventtrig.rst | 33 + docs/extension.rst | 33 + docs/foreign.rst | 142 ++ docs/function.rst | 65 + docs/index.rst | 132 ++ docs/indexes.rst | 43 + docs/install.rst | 182 +++ docs/issues.rst | 114 ++ docs/language.rst | 35 + docs/operator.rst | 36 + docs/operclass.rst | 36 + docs/operfamily.rst | 35 + docs/overview.rst | 117 ++ docs/predefaug.rst | 93 ++ docs/rule.rst | 35 + docs/schema.rst | 57 + docs/table.rst | 178 +++ docs/testing.rst | 175 +++ docs/textsearch.rst | 102 ++ docs/trigger.rst | 35 + docs/type.rst | 120 ++ docs/yamltodb.rst | 129 ++ pyrseas/.vscode/launch.json | 12 + pyrseas/__init__.py | 1 + pyrseas/augment/__init__.py | 31 + pyrseas/augment/audit.py | 63 + pyrseas/augment/column.py | 71 + pyrseas/augment/function.py | 116 ++ pyrseas/augment/schema.py | 97 ++ pyrseas/augment/table.py | 76 ++ pyrseas/augment/trigger.py | 64 + pyrseas/augmentdb.py | 145 ++ pyrseas/cmdargs.py | 110 ++ pyrseas/config.py | 61 + pyrseas/config.yaml | 160 +++ pyrseas/database.py | 650 +++++++++ pyrseas/dbaugment.py | 48 + pyrseas/dbobject/__init__.py | 680 ++++++++++ pyrseas/dbobject/cast.py | 174 +++ pyrseas/dbobject/collation.py | 93 ++ pyrseas/dbobject/column.py | 289 ++++ pyrseas/dbobject/constraint.py | 782 +++++++++++ pyrseas/dbobject/conversion.py | 114 ++ pyrseas/dbobject/dbtype.py | 740 +++++++++++ pyrseas/dbobject/eventtrig.py | 142 ++ pyrseas/dbobject/extension.py | 137 ++ pyrseas/dbobject/foreign.py | 693 ++++++++++ pyrseas/dbobject/function.py | 732 +++++++++++ pyrseas/dbobject/index.py | 353 +++++ pyrseas/dbobject/language.py | 140 ++ pyrseas/dbobject/operator.py | 236 ++++ pyrseas/dbobject/operclass.py | 250 ++++ pyrseas/dbobject/operfamily.py | 119 ++ pyrseas/dbobject/privileges.py | 203 +++ pyrseas/dbobject/rule.py | 143 ++ pyrseas/dbobject/schema.py | 371 ++++++ pyrseas/dbobject/table.py | 985 ++++++++++++++ pyrseas/dbobject/textsearch.py | 390 ++++++ pyrseas/dbobject/trigger.py | 282 ++++ pyrseas/dbobject/view.py | 226 ++++ pyrseas/dbtoyaml.py | 57 + pyrseas/lib/__init__.py | 0 pyrseas/lib/dbconn.py | 135 ++ pyrseas/lib/dbutils.py | 123 ++ pyrseas/testutils.py | 412 ++++++ pyrseas/yamltodb.py | 80 ++ pyrseas/yamlutil.py | 21 + requirements.txt | 2 + setup.cfg | 2 + setup.py | 66 + tests/__init__.py | 1 + tests/augment/test_audit.py | 274 ++++ tests/dbobject/__init__.py | 1 + tests/dbobject/test_cast.py | 171 +++ tests/dbobject/test_collation.py | 132 ++ tests/dbobject/test_column.py | 410 ++++++ tests/dbobject/test_constraint.py | 1275 ++++++++++++++++++ tests/dbobject/test_conversion.py | 109 ++ tests/dbobject/test_domain.py | 119 ++ tests/dbobject/test_eventtrig.py | 120 ++ tests/dbobject/test_extension.py | 158 +++ tests/dbobject/test_extern_file.py | 268 ++++ tests/dbobject/test_foreign.py | 395 ++++++ tests/dbobject/test_function.py | 522 ++++++++ tests/dbobject/test_index.py | 376 ++++++ tests/dbobject/test_matview.py | 115 ++ tests/dbobject/test_operator.py | 133 ++ tests/dbobject/test_operclass.py | 160 +++ tests/dbobject/test_operfamily.py | 85 ++ tests/dbobject/test_owner.py | 117 ++ tests/dbobject/test_privs.py | 446 +++++++ tests/dbobject/test_rule.py | 191 +++ tests/dbobject/test_schema.py | 134 ++ tests/dbobject/test_sequence.py | 180 +++ tests/dbobject/test_static.py | 103 ++ tests/dbobject/test_table.py | 436 ++++++ tests/dbobject/test_tablespace.py | 104 ++ tests/dbobject/test_textsearch.py | 276 ++++ tests/dbobject/test_trigger.py | 488 +++++++ tests/dbobject/test_type.py | 250 ++++ tests/dbobject/test_view.py | 217 +++ tests/functional/__init__.py | 1 + tests/functional/autodoc-schema.sql | 125 ++ tests/functional/film-schema-0.1.sql | 5 + tests/functional/film-schema-0.2.sql | 3 + tests/functional/film-schema-0.3a.sql | 8 + tests/functional/film-schema-0.3b.sql | 34 + tests/functional/pagila-schema.sql | 1748 +++++++++++++++++++++++++ tests/functional/test_autodoc.py | 69 + tests/functional/test_autodoc_dir.py | 81 ++ tests/functional/test_filmversions.py | 175 +++ tests/functional/test_pagila.py | 88 ++ tests/test_config.py | 81 ++ tox.ini | 34 + 142 files changed, 25396 insertions(+) create mode 100644 .gitignore create mode 100644 .readthedocs.yaml create mode 100644 .travis.yml create mode 100644 AUTHORS.rst create mode 100644 Changelog.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 META.json create mode 100644 Makefile create mode 100644 README.rst create mode 100644 docs/Makefile create mode 100644 docs/augmentdb.rst create mode 100644 docs/augobjects.rst create mode 100644 docs/cast.rst create mode 100644 docs/cfgobjects.rst create mode 100644 docs/cmdargs.rst create mode 100644 docs/collation.rst create mode 100644 docs/column.rst create mode 100644 docs/conf.py create mode 100644 docs/config.rst create mode 100644 docs/configitems.rst create mode 100644 docs/constraint.rst create mode 100644 docs/conversion.rst create mode 100644 docs/database.rst create mode 100644 docs/dbaugment.rst create mode 100644 docs/dbobject.rst create mode 100644 docs/dbtoyaml.rst create mode 100644 docs/devel.rst create mode 100644 docs/eventtrig.rst create mode 100644 docs/extension.rst create mode 100644 docs/foreign.rst create mode 100644 docs/function.rst create mode 100644 docs/index.rst create mode 100644 docs/indexes.rst create mode 100644 docs/install.rst create mode 100644 docs/issues.rst create mode 100644 docs/language.rst create mode 100644 docs/operator.rst create mode 100644 docs/operclass.rst create mode 100644 docs/operfamily.rst create mode 100644 docs/overview.rst create mode 100644 docs/predefaug.rst create mode 100644 docs/rule.rst create mode 100644 docs/schema.rst create mode 100644 docs/table.rst create mode 100644 docs/testing.rst create mode 100644 docs/textsearch.rst create mode 100644 docs/trigger.rst create mode 100644 docs/type.rst create mode 100644 docs/yamltodb.rst create mode 100644 pyrseas/.vscode/launch.json create mode 100644 pyrseas/__init__.py create mode 100644 pyrseas/augment/__init__.py create mode 100644 pyrseas/augment/audit.py create mode 100644 pyrseas/augment/column.py create mode 100644 pyrseas/augment/function.py create mode 100644 pyrseas/augment/schema.py create mode 100644 pyrseas/augment/table.py create mode 100644 pyrseas/augment/trigger.py create mode 100644 pyrseas/augmentdb.py create mode 100644 pyrseas/cmdargs.py create mode 100644 pyrseas/config.py create mode 100644 pyrseas/config.yaml create mode 100644 pyrseas/database.py create mode 100644 pyrseas/dbaugment.py create mode 100644 pyrseas/dbobject/__init__.py create mode 100644 pyrseas/dbobject/cast.py create mode 100644 pyrseas/dbobject/collation.py create mode 100644 pyrseas/dbobject/column.py create mode 100644 pyrseas/dbobject/constraint.py create mode 100644 pyrseas/dbobject/conversion.py create mode 100644 pyrseas/dbobject/dbtype.py create mode 100644 pyrseas/dbobject/eventtrig.py create mode 100644 pyrseas/dbobject/extension.py create mode 100644 pyrseas/dbobject/foreign.py create mode 100644 pyrseas/dbobject/function.py create mode 100644 pyrseas/dbobject/index.py create mode 100644 pyrseas/dbobject/language.py create mode 100644 pyrseas/dbobject/operator.py create mode 100644 pyrseas/dbobject/operclass.py create mode 100644 pyrseas/dbobject/operfamily.py create mode 100644 pyrseas/dbobject/privileges.py create mode 100644 pyrseas/dbobject/rule.py create mode 100644 pyrseas/dbobject/schema.py create mode 100644 pyrseas/dbobject/table.py create mode 100644 pyrseas/dbobject/textsearch.py create mode 100644 pyrseas/dbobject/trigger.py create mode 100644 pyrseas/dbobject/view.py create mode 100644 pyrseas/dbtoyaml.py create mode 100644 pyrseas/lib/__init__.py create mode 100644 pyrseas/lib/dbconn.py create mode 100644 pyrseas/lib/dbutils.py create mode 100644 pyrseas/testutils.py create mode 100644 pyrseas/yamltodb.py create mode 100644 pyrseas/yamlutil.py create mode 100644 requirements.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/augment/test_audit.py create mode 100644 tests/dbobject/__init__.py create mode 100644 tests/dbobject/test_cast.py create mode 100644 tests/dbobject/test_collation.py create mode 100644 tests/dbobject/test_column.py create mode 100644 tests/dbobject/test_constraint.py create mode 100644 tests/dbobject/test_conversion.py create mode 100644 tests/dbobject/test_domain.py create mode 100644 tests/dbobject/test_eventtrig.py create mode 100644 tests/dbobject/test_extension.py create mode 100644 tests/dbobject/test_extern_file.py create mode 100644 tests/dbobject/test_foreign.py create mode 100644 tests/dbobject/test_function.py create mode 100644 tests/dbobject/test_index.py create mode 100644 tests/dbobject/test_matview.py create mode 100644 tests/dbobject/test_operator.py create mode 100644 tests/dbobject/test_operclass.py create mode 100644 tests/dbobject/test_operfamily.py create mode 100644 tests/dbobject/test_owner.py create mode 100644 tests/dbobject/test_privs.py create mode 100644 tests/dbobject/test_rule.py create mode 100644 tests/dbobject/test_schema.py create mode 100644 tests/dbobject/test_sequence.py create mode 100644 tests/dbobject/test_static.py create mode 100644 tests/dbobject/test_table.py create mode 100644 tests/dbobject/test_tablespace.py create mode 100644 tests/dbobject/test_textsearch.py create mode 100644 tests/dbobject/test_trigger.py create mode 100644 tests/dbobject/test_type.py create mode 100644 tests/dbobject/test_view.py create mode 100644 tests/functional/__init__.py create mode 100644 tests/functional/autodoc-schema.sql create mode 100644 tests/functional/film-schema-0.1.sql create mode 100644 tests/functional/film-schema-0.2.sql create mode 100644 tests/functional/film-schema-0.3a.sql create mode 100644 tests/functional/film-schema-0.3b.sql create mode 100644 tests/functional/pagila-schema.sql create mode 100644 tests/functional/test_autodoc.py create mode 100644 tests/functional/test_autodoc_dir.py create mode 100644 tests/functional/test_filmversions.py create mode 100644 tests/functional/test_pagila.py create mode 100644 tests/test_config.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed7ff9f --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.pyc +*~ +MANIFEST +build +dist +docs/_build +Pyrseas.egg-info +.coverage +htmlcov +.tox +.cache +/*.egg +venv diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..e336df8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: requirements.txt + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5f2fe43 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,50 @@ +language: python + +addons: + apt: + update: true + packages: postgresql-common + sources: + - sourceline: deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main + +jobs: + include: + - python: "2.7" + env: POSTGRES=9.4 + - python: "2.7" + env: POSTGRES=9.5 + - python: "2.7" + env: POSTGRES=9.6 + - python: "3.6" + env: POSTGRES=10 + - python: "3.7" + env: POSTGRES=11 + - python: "3.8" + env: POSTGRES=12 + +# Ensure the desired version of Postgres is installed and running +# Most of this is normally handled by Travis automatically, but only for certain versions of Postgres +# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash +before_install: + - sudo systemctl stop postgresql + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install postgresql-$POSTGRES postgresql-plperl-$POSTGRES postgresql-plpython3-$POSTGRES + # the port may have been auto-configured to use 5433 if it thought 5422 was already in use + - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf + # postgresql-11+ default to "peer" + - sudo sed -i -E -e 's/^local\s+all\s+postgres\s+peer/local all postgres trust/' /etc/postgresql/*/main/pg_hba.conf + - sudo mkdir -p /var/ramfs/postgresql + - if [ ! -d /var/ramfs/postgresql/$POSTGRES ]; then sudo cp -rp /var/lib/postgresql/$POSTGRES /var/ramfs/postgresql/$POSTGRES; fi + - sudo systemctl start postgresql@$POSTGRES-main + - (cd /; sudo -u postgres createuser -e -s travis || echo failed createuser) + - (cd /; sudo -u postgres createdb -e -O travis travis || echo failed created) + +# Ensure Postgres is configured as described in docs/testing.rst +before_script: + - sudo locale-gen --no-archive fr_FR.UTF-8 + - sudo mkdir -p /extra/pg/ts1 /extra/pg/ts2 + - sudo chown postgres:postgres /extra/pg/ts1 /extra/pg/ts2 + - psql -Upostgres -c "CREATE TABLESPACE ts1 LOCATION '/extra/pg/ts1'" + - psql -Upostgres -c "CREATE TABLESPACE ts2 LOCATION '/extra/pg/ts2'" + +script: + - python setup.py test diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..1b48fb8 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,29 @@ +Pyrseas was started in 2010. + +The PRIMARY AUTHORS are or have been: + + * Joe Abbate + * Ronan Dunklau + * Douglas Fraser + * Mark Goldfinch + * Ryan Graham + * Andjelko Horvat + * Roger Hunwicks + * Ghislain Leveque + * Chris Mayo + * Philippe Pepiot + * Andrey Popp + * Ross J. Reedstrom + * github.com/shirkey + * Denis Smirnov + * Feike Steenbergen + * Paulo Tioseco + * Daniele Varrazzo + * Vasiliy Yeremeyev + +A big THANK YOU goes to the following individuals that provided +inspiration for some of the features and design of Pyrseas: + + * Ken Downs for creating the Andromeda project. + + * Robert Brewer for Post-Facto. diff --git a/Changelog.rst b/Changelog.rst new file mode 100644 index 0000000..3f45a81 --- /dev/null +++ b/Changelog.rst @@ -0,0 +1,347 @@ +0.10.0 (7-Nov-2022) + +Convert Pyrseas to use Psycopg version 3. Remove support for Python +2.x and Postgres versions prior to PG 10. + +Reinstate code for interfacing with psycopg, removing the dependency +on pgdbconn. + +Fixed various issues, including: + + * Partial fix for database objects owned by extensions. (#236) + + * Deal with pg_pltemplate catalog being removed in PG 13. (#226, #227) + + * Problem with trigger procedures with arguments. (#219) + +0.9.1 (14-Apr-2020) + +Expanded the number of keywords that are quoted if they are present in +generated SQL statements. (#212) + +Various changes to support Postgres 12. + +Fixed various issues, including: + + * Handling of multiple foreign key constraints (#210) + + * Column qualifications in Extensions query (#208) + +0.9.0 (22-Jul-2019) + +Schema 'public' is no longer treated as a special case (176) + +Yamltodb was changed to support Postgres 11, in particular due +to changes to the 'pg_proc' catalog (#195) + +Dbtoyaml now outputs column information for views, both regular and +materialized (184) + +Fixed various issues, including: + + * Do not assume constraints always refer to columns (#188) + + * Add non-'public' schemas to search_path to avoid problems with + some extensions (e.g., PostGIS) that need them to resolve + objects (#91) + + * Prevent a view definition from changing a column type (#90) + + * Inconsistent handling of FOREIGN KEY and UNIQUE constraints + leading to AttributeErrors (#182) + + * Recreation of tsvector triggers when columns are added (#179) + +0.8.0 (12-Dec-2017) + +Significant rearchitecture of methods to generate SQL. + + * An object dependency graph is built and traversed to generate SQL + in correct order (#72, #86, #100) + +Added support for Postgres 10, specifically: + + * Table partitioning syntax (#163) + + * Column specification GENERATED AS IDENTITY (#164) + +Added support for other Postgres features: + + * Parallel safe functions and partial aggregation (#161) + + * RANGE types (#173) + + * ALTER TYPE ADD VALUE for changes to ENUM types (#87) + + +0.7.2 (23-Jan-2015) + + Fixed various issues, including: + + * Do not error on tables whose names start with 'public' (#109) + + * Deal properly with inherited constraints in children tables (#102) + + * Handle external languages like plv8 correctly (#97) + + * Correct quoting of mixed case constraint names (#83) + + * Avoid problems with certain complex index definitions (#98) + + * Have dbtoyaml output correctly a table with an embedded period in + the name and having an associated sequence (#79) + + * Use relative paths in database summary for ``--multiple-files`` + (#93) + + * Support mapping of indexes on materialized views (#82) + +0.7.1 (5-Dec-2013) + + * Moved ``config.yaml`` under ``pyrseas`` directory and use + ``package_data`` to install (#77) + + * yamltodb output to a file is encoded using utf-8 (#78) + + +0.7.0 (25-Nov-2013) + + * Added support for: + + - Postgres 9.3, specifically + + + EVENT TRIGGER + + MATERIALIZED VIEWS + + - CLUSTER + - Partial indexes + - Storage parameters in CREATE and ALTER TABLE + - ALTER COLUMN SET STATISTICS + - LEAKPROOF qualifier for FUNCTIONs + - YAML multi-line string formatting for view definitions, + function source text and object comments + + * Configuration files + + All Pyrseas utilities can now use YAML-formatted configuration + files, in addition to command line options + + * Multiple-file input or output + + Spread database object information across a version control + repository + + * Data export/import + + Load a database with static data in production or data subsets + for testing + + * dbtoyaml/yamltodb + + - Added --quote-reserved option to yamltodb + - Exclude arguments from sfunc and finalfunc attributes of + aggregate functions (#54) + - Correct generation of SQL for functions with DEFAULT + arguments (#52) + + * Augmenter + + New utility (dbaugment) to consistently add objects to an + existent database. This is currently an experimental + feature and covers adding audit columns to tables. + + * TTM-inspired relational interface + + A new interface to Postgres, inspired by *The Third Manifesto* + + +0.6.1 (31-Jan-2013) + + * Add support for INSTEAD OF triggers on views (#50). + + * Eliminated yamltodb generation of spurious REVOKE/GRANT commands + (#51). + + * Removed setuptools from setup.py install_requires. + + +0.6.0 (26-Oct-2012) + + * Added support for: + + - EXTENSIONs + - COLLATIONs + - OWNER information + - Access privileges (GRANT and REVOKE) + - TABLESPACEs for tables, primary keys and indexes + - MATCH attributes for foreign keys (#34) + - ALTER composite TYPE ADD/DROP/RENAME ATTRIBUTE + - ENUMs with no labels (#31) + - UNLOGGED tables (#45) + - CREATE FUNCTION SET configuration_parameter (#46) + - PostgreSQL 9.2 + + * Correctly support index functions/expressions (#3, #44). + + * Schema-qualify composite types when dropping or renaming + attributes (#47) + + * Fix DbConnection exception handling under Python 3 (#25). + + * dbtoyaml + + - Fix -t option to output sequences owned by table and the schema + description. + - Use pg_user_mappings view to allow usage by non-superusers. + + * yamltodb + + - Schema-qualify table when dropping columns (#26). + - Correct column drop/add case in middle of table (#8). + - Fix adding and dropping of columns in inherited tables (#33). + - Enable renaming of indexes (#38). + - Ignore all temp schemas (#37) + + * dbtoyaml/yamltodb + + - Give PGUSER precedence over USER environment variable. + + * Testing + + - Added support, via Tox, for testing against multiple + PostgreSQL/Python combinations + + - Changes and documentation for testing on Microsoft Windows + + +0.5.0 (10-Mar-2012) + + * Added support for: + + - TEXTSEARCH parsers, dictionaries, configurations and templates + - FOREIGN DATA WRAPPERs, SERVERs, USER MAPPINGs and FOREIGN TABLEs + - ROWS clause in set-returning functions (issue #11) + - Deferrable/deferred constraints (#13) + - CATEGORY and PREFERRED clauses for TYPEs, + SORTOP clause for AGGREGATEs + HASHES and MERGES clauses for OPERATORs (#15) + - Operator class qualifiers for INDEXes (#16) + - Python 3.2 and later + + * Correct schema normalization for constraints (#9) and indexes. + + * Fix COMMENTs generated for constraints (#12). + + * Fix DEFAULT clause for OPERATOR CLASS. + + * dbtoyaml + + - When restricting to specific schemas or tables, include + non-schema objects (e.g., languages). + + * yamltodb + + - Add -n/--schema option (#6). + - Add -u/--update option to apply SQL statements to target + database. + - Exclude database-wide objects when -n/--schema is used (#21). + - Allow YAML spec argument to be read from standard input. + + * dbtoyaml/yamltodb + + - Add -o/--output option + - Add -W/--password option (#18) + + +0.4.1 (27-Oct-2011) + + * Make the initial SET search_path persistent. + + * Correct exclusion of PG internal schemas in various queries. + + * Fix generation of COMMENTs with single quotes in the text. + + * For inherited tables, only generate constraints that are defined + locally. + + * Correct generation of ALTER TABLE ADD/DROP COLUMN when input + columns are in different order than original. + + * Support PG 9.1 (add description for PL/pgSQL language). + + +0.4.0 (26-Sep-2011) + + * Added support for: + + - CASTs + - CONSTRAINT TRIGGERs + - CONVERSIONs + - OPERATORs, OPERATOR CLASSes and OPERATOR FAMILies + - Dynamically loaded C language functions + - Composite and base TYPEs + + * Clean up and enhance documentation and redundant methods. + + * Use obj_description/col_description functions instead of querying + pg_description directly. + + +0.3.1 (26-Aug-2011) + + * Added workaround for incorrect assumption that 'public' schema is + always present (issue #4). + + * Added support for delimited (or quoted) identifiers, e.g., those + with embedded spaces, upper case characters, etc. (except for SQL + keywords) (issue #5). + + +0.3.0 (30-Jun-2011) + + * Added support for: + + - AGGREGATE functions + - DOMAINs + - ENUMerated TYPEs + - Functions returning table row types + - INDEXes on expressions (issue #3) + - Rewrite RULEs + - SECURITY DEFINER functions + - TRIGGERs + + * Enhanced host/port defaults to use sockets, resulting in noticeable + performance improvement. + + +0.2.1 (7-Jun-2011) + + * Fixed problem with mapping a FOREIGN KEY in a table with a dropped + column (issue #2). + + +0.2.0 (19-May-2011) + + * Added support for: + + - COMMENTs on schemas, tables, columns and functions + - FOREIGN KEY ON UPDATE and ON DELETE actions + - ALTER TABLE RENAME COLUMN and enhanced support for other ALTER + object RENAME statements. + - VIEWs + - INHERITed tables, and by extension, partitioned tables. + - PROCEDURAL LANGUAGEs + - FUNCTIONs. + + * Added files for release via PGXN. + * Added support for testing against multiple PostgreSQL versions. + * Fixed cross-schema REFERENCES failure in dbtoyaml (issue #1). + + +0.1.0 (5-Apr-2011) + + * Initial release + + - dbtoyaml and yamltodb support PostgreSQL schemas, tables, + sequences, check constraints, primary keys, foreign keys, unique + constraints and indexes. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4a47a19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2010 by Joe Abbate, see AUTHORS for more details. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Pyrseas project nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f68f227 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include README.rst AUTHORS.rst Changelog.rst LICENSE Makefile META.json +recursive-include tests *.py +recursive-include docs * +prune docs/_build diff --git a/META.json b/META.json new file mode 100644 index 0000000..22470e1 --- /dev/null +++ b/META.json @@ -0,0 +1,57 @@ +{ + "name": "Pyrseas", + "abstract": "Utilities to assist in database schema versioning", + "description": "Pyrseas provides utilities to describe a PostgreSQL database schema as YAML, to verify the schema against the same or a different database and to generate SQL that will modify the schema to match the YAML description. Supports PostgreSQL version 10 through 15.", + "version": "0.10.0", + "maintainer": "Joe Abbate ", + "license": "bsd", + "prereqs": { + "runtime": { + "requires": { + "PostgreSQL": "9.4.0" + } + } + }, + "provides": { + "augmentdb": { + "file": "pyrseas/augmentdb.py", + "docfile": "docs/augmentdb.rst", + "version": "0.10.0", + "abstract": "Generates an augmented YAML description of a Postgres database from its catalogs and an augmentation specification" + }, + "dbtoyaml": { + "file": "pyrseas/dbtoyaml.py", + "docfile": "docs/dbtoyaml.rst", + "version": "0.10.0", + "abstract": "Outputs a YAML description of a Postgres database's tables and other objects (metadata), suitable for storing in a version control repository" + }, + "yamltodb": { + "file": "pyrseas/yamltodb.py", + "docfile": "docs/yamltodb.rst", + "version": "0.10.0", + "abstract": "Generates SQL statements to modify a database so that it will match an input YAML/JSON specification" + } + }, + "resources": { + "homepage": "https://perseas.github.io/", + "bugtracker": { + "web": "https://github.com/perseas/Pyrseas/issues" + }, + "repository": { + "url": "git://github.com/perseas/Pyrseas.git", + "web": "https://github.com/perseas/Pyrseas", + "type": "git" + } + }, + "generated_by": "Joe Abbate", + "meta-spec": { + "version": "1.0.0", + "url": "http://pgxn.org/meta/spec.txt" + }, + "tags": [ + "version control", + "yaml", + "database version control", + "schema versioning" + ] +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8700b09 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +# +# Pyrseas Makefile +# + +PYTHON = python3 + +.PHONY: all build docs install installcheck check clean + +all: + $(PYTHON) setup.py build + +build: + $(PYTHON) setup.py sdist --format=gztar,zip + $(PYTHON) setup.py bdist_wheel + +docs: + $(MAKE) -C docs html + +install: + $(PYTHON) setup.py install + +installcheck check: + $(PYTHON) setup.py test + +clean: + $(MAKE) -C docs clean + $(PYTHON) setup.py clean diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1857d8b --- /dev/null +++ b/README.rst @@ -0,0 +1,43 @@ +======= +Pyrseas +======= + +.. image:: https://api.travis-ci.org/perseas/Pyrseas.png?branch=master + :target: https://travis-ci.com/perseas/Pyrseas + +Pyrseas provides utilities to describe a PostgreSQL database schema as +YAML, to verify the schema against the same or a different database +and to generate SQL that will modify the schema to match the YAML +description. + +Features +-------- + +- Outputs a YAML description of a Postgres database's tables + and other objects (metadata), suitable for storing in a version + control repository + +- Generates SQL statements to modify a database so that it will match + an input YAML/JSON specification + +- Generates an augmented YAML description of a Postgres database + from its catalogs and an augmentation specification. + +Requirements +------------ + +- PostgreSQL 10 or higher + +- Python 3.7 or higher + +License +------- + +Pyrseas is free (libre) software and is distributed under the BSD +license. Please see the LICENSE file for details. + +Documentation +------------- + +Please visit `Read the Docs `_ +for the latest documentation. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..eef3aaa --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Pyrseas.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Pyrseas.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/augmentdb.rst b/docs/augmentdb.rst new file mode 100644 index 0000000..16dcd13 --- /dev/null +++ b/docs/augmentdb.rst @@ -0,0 +1,30 @@ +Augmenter Databases +=================== + +.. module:: pyrseas.augmentdb + +The :mod:`augmentdb` module defines the class :class:`AugmentDatabase`. + +Augmenter Database +------------------ + +An :class:`AugmentDatabase` is derived from +:class:`~pyrseas.database.Database`. It contains two "dictionary" +objects. + +One is the :class:`Dicts` container from its parent class. The `db` +Dicts object, defines the database schemas, including their tables and +other objects, by querying the system catalogs. + +The second container is an :class:`AugDicts` object. The `adb` +AugDicts object specifies the schemas to be augmented and the +augmenter configuration objects. The latter objects may be supplied +either by other Augmenter modules or from the ``augmenter`` +configuration tree on the `aug_map` supplied to the :meth:`apply` +method. + +.. autoclass:: AugmentDatabase + +.. automethod:: AugmentDatabase.apply + +.. automethod:: AugmentDatabase.from_augmap diff --git a/docs/augobjects.rst b/docs/augobjects.rst new file mode 100644 index 0000000..c38ead0 --- /dev/null +++ b/docs/augobjects.rst @@ -0,0 +1,42 @@ +Augmentation Objects +==================== + +These objects are defined in the `aug_map` argument to the `apply` +method of :class:`~pyrseas.augmentdb.AugmentDatabase`. They tie the +desired augmentations, e.g., audit columns, to the tables to be +affected, and the schemas owning the tables. + +.. module:: pyrseas.augment.schema + +Augmentation Schema +------------------- + +.. autoclass:: AugSchema + +.. automethod:: AugSchema.apply + +.. autoclass:: AugSchemaDict + +.. automethod:: AugSchemaDict.from_map + +.. automethod:: AugSchemaDict.link_current + +.. automethod:: AugSchemaDict.link_refs + + +.. module:: pyrseas.augment.table + +Augmentation Table +------------------ + +.. autoclass:: AugDbClass + +.. autoclass:: AugTable + +.. automethod:: AugTable.apply + +.. autoclass:: AugClassDict + +.. automethod:: AugClassDict.from_map + +.. automethod:: AugClassDict.link_current diff --git a/docs/cast.rst b/docs/cast.rst new file mode 100644 index 0000000..b52f794 --- /dev/null +++ b/docs/cast.rst @@ -0,0 +1,39 @@ +Casts +===== + +.. module:: pyrseas.dbobject.cast + +The :mod:`cast` module defines two classes, :class:`Cast` and +:class:`CastDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Cast +---- + +:class:`Cast` is derived from :class:`~pyrseas.dbobject.DbObject` and +represents a `Postgres cast +`_. + +A cast is identified externally as ``cast ( AS +)``. + +.. autoclass:: Cast + +.. automethod:: Cast.extern_key + +.. automethod:: Cast.identifier + +.. automethod:: Cast.to_map + +.. automethod:: Cast.create + +Cast Dictionary +--------------- + +:class:`CastDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of casts in a database. + +.. autoclass:: CastDict + +.. automethod:: CastDict.from_map diff --git a/docs/cfgobjects.rst b/docs/cfgobjects.rst new file mode 100644 index 0000000..81f0367 --- /dev/null +++ b/docs/cfgobjects.rst @@ -0,0 +1,97 @@ +Augmenter Configuration Objects +=============================== + +These configuration objects are predefined in the Augmenter modules or +can be defined or overridden by configuration elements in the +``augmenter`` map. Please see also :doc:`configitems` and +:doc:`predefaug`. + +.. module:: pyrseas.augment.function + +Configuration Functions +----------------------- + +A :class:`CfgFunction` class specifies a Postgres function to be used +by other augmenter objects. For example, this includes procedures to +be invoked by triggers used to maintain audit columns. The +:class:`CfgFunctionDict` class holds all the :class:`CfgFunction` +objects, indexed by the function name and its arguments. A +:class:`CfgFunctionSource` class represents the source code for a +function or part of that source code. A :class:`CfgFunctionTemplate` +class represents the source code for a function, which may include +other elements that can be substituted in the final result. The class +:class:`CfgFunctionSourceDict` holds all the templates currently +defined. + +.. autoclass:: CfgFunction + +.. automethod:: CfgFunction.apply + +.. autoclass:: CfgFunctionDict + +.. automethod:: CfgFunctionDict.from_map + +.. autoclass:: CfgFunctionSource + +.. autoclass:: CfgFunctionTemplate + +.. autoclass:: CfgFunctionSourceDict + + +.. module:: pyrseas.augment.column + +Configuration Columns +--------------------- + +A :class:`CfgColumn` class defines a column to be added to a table by +other augmenter objects. For example, this includes various columns +that serve to capture audit trail information. The columns can be +combined in various ways by the :class:`CfgAuditColumn` objects. The +:class:`CfgColumnDict` class holds all the :class:`CfgColumn` objects, +indexed by column name. + +.. autoclass:: CfgColumn + +.. automethod:: CfgColumn.apply + +.. autoclass:: CfgColumnDict + +.. automethod:: CfgColumnDict.from_map + + +.. module:: pyrseas.augment.trigger + +Configuration Triggers +---------------------- + +A :class:`CfgTrigger` class defines a trigger to be added to a table +by other augmentation objects. For example, this includes triggers to +maintain audit trail columns. The :class:`CfgTriggerDict` class holds +all the :class:`CfgTrigger` objects, indexed by trigger name. + +.. autoclass:: CfgTrigger + +.. automethod:: CfgTrigger.apply + +.. autoclass:: CfgTriggerDict + +.. automethod:: CfgTriggerDict.from_map + + +.. module:: pyrseas.augment.audit + +Configuration Audit Columns +--------------------------- + +A :class:`CfgAuditColumn` class defines a set of attributes (columns, +triggers) to be added to a table. The :class:`CfgAuditColumnDict` +class holds all the :class:`CfgAuditColumn` objects, indexed by +augmentation name. + +.. autoclass:: CfgAuditColumn + +.. automethod:: CfgAuditColumn.apply + +.. autoclass:: CfgAuditColumnDict + +.. automethod:: CfgAuditColumnDict.from_map diff --git a/docs/cmdargs.rst b/docs/cmdargs.rst new file mode 100644 index 0000000..3cdfca2 --- /dev/null +++ b/docs/cmdargs.rst @@ -0,0 +1,91 @@ +Common Command Line Options +=========================== + +The Pyrseas utilities support the following command line options: + +.. cmdoption:: -c + --config + + Specifies an additional `configuration file` to be read and merged + with configuration information from other sources. See + :doc:`config` for more details. + +.. cmdoption:: -H + --host + + Specifies the `host name` of the machine on which the Postgres + server is running. The default host name is determined by + Postgres (normally, a Unix-domain socket or ``localhost``). + +.. cmdoption:: -h, --help + + Show help about the program's command line arguments, and exit. + +.. cmdoption:: -o + --output + + Send output to the specified `file`. If this is omitted, the + standard output is used. + +.. cmdoption:: -p + --port + + Specifies the `port` on which the Postgres server is listening + for connections. The default port number is determined by + Postgres (normally, 5432). + +.. cmdoption:: -r + --repository + + Specifies the `path` to a directory where metadata and static data + files will be written to or read from, or where an additional + configuration file can be found. Normally, this will be the root + of a version control repository. If this is not specified on the + command line or in a configuration file, it defaults to the + current working directory. + +.. cmdoption:: -U + --user + + Postgres `user name` to connect as. The default user name is + determined by Postgres (normally, the name of the operating system + user running the program). + +.. cmdoption:: --version + + Print the program name and version identifier and exit. + +.. cmdoption:: -W, --password + + Force the program to prompt for a password before connecting to a + database. If this option is not specified and password + authentication is required, the program will resort to libpq + defaults, i.e., `password file + `_ + or `PGPASSWORD environment variable + `_. + +Short options (those only one character long) can be concatenated with +their value arguments, e.g.:: + + dbtoyaml -p5433 dbname + +Several short options can be joined together, using only a single - +prefix, as long as only the last option (or none of them) requires a +value. + +Long options (those with names longer than a single-character) can be +separated from their arguments by a '=' or passed as two separate +arguments. For example:: + + dbtoyaml --port=5433 dbname + +or:: + + dbtoyaml --port 5433 dbname + +Long options can be abbreviated as long as the abbreviation is +unambiguous:: + + dbtoyaml --pass dbname + diff --git a/docs/collation.rst b/docs/collation.rst new file mode 100644 index 0000000..1071180 --- /dev/null +++ b/docs/collation.rst @@ -0,0 +1,31 @@ +Collations +========== + +.. module:: pyrseas.dbobject.collation + +The :mod:`collation` module defines two classes, :class:`Collation` +and :class:`CollationDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Collation +--------- + +:class:`Collation` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +collation +`_. + +.. autoclass:: Collation + +.. automethod:: Collation.create + +Collation Dictionary +-------------------- + +:class:`CollationDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of collations in a database. + +.. autoclass:: CollationDict + +.. automethod:: CollationDict.from_map diff --git a/docs/column.rst b/docs/column.rst new file mode 100644 index 0000000..6b3f4bc --- /dev/null +++ b/docs/column.rst @@ -0,0 +1,52 @@ +Columns +======= + +.. module:: pyrseas.dbobject.column + +The :mod:`column` module defines two classes, :class:`Column` derived +from :class:`DbSchemaObject` and :class:`ColumnDict`, derived from +:class:`DbObjectDict`. + +Column +------ + +:class:`Column` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a column of a +table, or an attribute of a composite type. Its :attr:`keylist` +attributes are the schema name and the table name. + +A :class:`Column` has the following attributes: :attr:`name`, +:attr:`type`, :attr:`not_null`, :attr:`default` and +:attr:`collation`. The :attr:`number` attribute is also present but is +not made visible externally. + +.. autoclass:: Column + +.. automethod:: Column.to_map + +.. automethod:: Column.add + +.. automethod:: Column.add_privs + +.. automethod:: Column.diff_privileges + +.. automethod:: Column.comment + +.. automethod:: Column.drop + +.. automethod:: Column.rename + +.. automethod:: Column.alter + +Column Dictionary +----------------- + +Class :class:`ColumnDict` is a dictionary derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of columns in a database, across multiple tables. It is indexed by the +schema name and table name, and each value is a list of +:class:`Column` objects. + +.. autoclass:: ColumnDict + +.. automethod:: ColumnDict.from_map diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..7f3f739 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Pyrseas documentation build configuration file, created by +# sphinx-quickstart on Fri Dec 17 22:06:15 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = "Pyrseas" +copyright = "2010-2022, Joe Abbate" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +import re +from pyrseas import __version__ as release +version = re.match(r'\d+\.\d+(?:\.\d+)?', release).group() +# The full version, including alpha/beta/rc tags. +if 'dev' in release: + release = release[:release.find('dev') + 3] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Pyrseasdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Pyrseas.tex', u'Pyrseas Documentation', + u'Joe Abbate', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 0000000..fb649d9 --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,88 @@ +Configuration +============= + +The Pyrseas utilities allow you to configure various options through a +number of YAML specification files, none of which are required--but +the system configuration file is provided by the normal installation. + +If a configuration parameter is specified in more than one file, the +latter file in the list of files below overrides any earlier +specification. Any configuration item specified on the command line +takes precedence over any such item in a configuration file. + +Configuration File Name +----------------------- + +The default configuration file name is ``config.yaml``. If desired, +you can override this with the environment variable +``PYRSEAS_CONFIG_FILE``, but be aware that this will affect all three +levels below. + +System Configuration +-------------------- + +The system configuration file is distributed with Pyrseas and is +normally installed in the ``pyrseas`` library directory. + +If desired, you can override this using the ``PYRSEAS_SYS_CONFIG`` +environment variable. This can be defined as a full path, including a +file name, or a directory location, in which case the default file +name as mentioned above under `Configuration File Name`_ will be +appended to the path. + +Currently, this file includes specifications for functions, triggers +and other objects used by the :program:`dbaugment` utility. It also +includes the default directory path for storing multiple YAML files in +a VCS repository, and the path to data files for use by the data +import and export facilities. + +User Configuration +------------------ + +Each user can have his or her own configuration file. The default +location for this depends on the platform. Under Linux, BSD, OS/X and +other Unix variants, place the file under your home directory, in the +subdirectory ``.config/pyrseas/``. Under Windows, put the file in +``%APPDATA%\pyrseas\``. + +You can override the location of the user configuration file using the +``PYRSEAS_USER_CONFIG`` environment variable. This can be defined as +a full path, including a file name, or a directory location, in which +case the default file name as mentioned above under `Configuration +File Name`_ will be appended to the path. + +If present, the user configuration file will be merged with the system +configuration. + +It is recommended that the user configuration file only be used for +non-project-specific purposes. For example, if you frequently use +Pyrseas against a remote database or on a non-standard port, you can +specify the host or port in your personal configuration file. + +Repository Configuration +------------------------ + +A configuration file can be placed in a version control repository or +project directory, so that it can be under version control together +with other Pyrseas files such as the output from ``dbtoyaml +--multiple-files``. The default location for the repository can be +specified in the user configuration, using the keys ``repository`` and +``path``, for example:: + + repository: + path: /home/user/project/repo + +You can also use the :option:`--repository` command line option to +specify (or override) the directory path to the root of the repository +and the utilities will look for a configuration file in that location. + +If present, the repository configuration file will be merged with the +system and user configuration information. + +Command Line Configuration +-------------------------- + +The utilities also allow you to specify a fourth configuration file on +the command line, using the :option:`--config` command line option. +Again, if the file exists, its information will be merged with +previously read files. diff --git a/docs/configitems.rst b/docs/configitems.rst new file mode 100644 index 0000000..127f283 --- /dev/null +++ b/docs/configitems.rst @@ -0,0 +1,175 @@ +Configuration Items +=================== + +The following lists the various sections allowed in a configuration +file and the items that are recognized by the Pyrseas utilities. + +Augmenter +--------- + +This section is used by the :program:`dbaugment` utility (see +:doc:`dbaugment`). Most of these are specified in the system +configuration file delivered with Pyrseas, but can also be included or +overridden in user or repository configuration files. + +- audit_columns: This section defines combinations of columns and + triggers to be added to tables. Both columns and triggers are + specified as YAML lists (to be consistent with :program:`dbtoyaml` + YAML output), although normally a single trigger will be necessary + per column combination. The columns and triggers should reference + previously defined items in the ``columns`` and ``triggers`` + sections (see below). See :doc:`predefaug` for audit columns + defined in the system ``config.yaml``. + +- columns: This section defines prototype columns to be added to a + table by Augmenter. For each column, a valid `Postgres data type + `_ + should be included. + + You can also add a ``not_null`` constraint and a ``default`` + specification. See :doc:`predefaug` for columns defined in the + system ``config.yaml``. In a repository or user configuration file, + you can also specify an alternate name for a previously defined + column. For example, if you prefer that the ``modified_timestamp`` + columns be named ``last_update``, you can add the following to a + configuration file:: + + augmenter: + columns: + modified_timestamp: + name: last_update + +- function_templates: This section defines the source text for the + trigger functions (see below) using a template language. Any text + enclosed in double braces, e.g., ``{{modified_by_user}}``, will be + replaced, typically by a previously defined column or its alternate + name (see above). + +- functions: This section defines prototype trigger functions to be + invoked by audit columns or other augmentations. The following + items can be specified for each function: + + - description: Text for a `COMMENT + `_ + statement on the function. + + - language: Procedural language, e.g., ``plpgsql``, in which the + function is written. + + - returns: Value should be ``trigger``. + + - security_definer: Indicates whether the function is to be executed + with the privileges of the user that created it. This is usually + needed for audit column trigger functions. + + - source: This is usually a reference to a function template (see + above) enclosed in double braces, e.g., + ``{{functempl_audit_default}}``. However, in user or repository + configurations, this can also be the actual text of the function. + + See :doc:`predefaug` for functions defined in the system + configuration file. + +- schema pyrseas: This section currently defines three functions that + may be installed in the ``pyrseas`` schema if the ``full`` audit + columns specifications is added for Augmenter processing. + +- schemas and tables: Multiple ``schema schema-name`` sections can be + present, typically in a repository configuration file. Each such + section can include ``table table-name`` items, and under each the + ``audit_columns`` specifications to be added to the given table. + For example:: + + augmenter: + schema public: + table t1: + audit_columns: default + +- triggers: This section defines the prototype triggers to be used + with audit columns and other augmentations. The following items can + be specified for each trigger: + + - events: This is a list that can include one or more of ``insert``, + ``update`` or ``delete`` (the latter is not used for audit columns + but may be used in future augmentations). + + - level: This can take the values ``row`` or ``statement`` (usually + the former). + + - name: This specifies the name to be given to a trigger. It can be + a template using ``{{table_name}}`` which will then be replaced + with the actual table name on which the trigger will act. + + - procedure: This is the invocation name, e.g., ``audit_default()`` + of the function to be called when the trigger fires. + + - timing: This can take the values ``before`` or ``after`` (usually + the former). + +Database +-------- + +This section is primarily for a user configuration file. If you +frequently connect to a particular host, port or as a given user, that +are *not* the Postgres defaults, adding corresponding entries to your +user configuration file allows you to automatically override the +defaults. If for a given invocation you need to connect to or as a +different host, port or user, you can still override the configuration +using the command line options (see :doc:`cmdargs`): + +- host: Name of the host to connect. Please refer to the `Postgres + connection host documentation + `_ + for details and defaults. + +- port: Port number to connect to. See the `Postgres connection port + documentation + `_ + for more. + +- username: Name of the user to connect as. View the `Postgres + connection user documentation + `_ + for more. + +Datacopy +-------- + +This section is normally in a user or repository configuration file. +It is used by :program:`dbtoyaml` and :program:`yamltodb` to determine +which tables should be exported from or imported to the database. It +consists of schema names, using the format `schema schema_name`, +followed by lists of table names. For example:: + + datacopy: + schema public: + - t1 + - t2 + schema s1: + - t3 + +Repository +---------- + +This section is used by all utilities (but :program:`dbaugment` does +not fully support it). The "repository" is intended to be a version +control, e.g., Git, Mercurial, or Subversion, repository. + +- data: Path, relative to the root of the repository, where + :program:`dbtoyaml` and :program:`yamltodb` place or expect the + files containing data exported from or imported to the database. The + tables to be exported or imported are specified in the ``Datacopy`` + section. The default value (defined in the system ``config.yaml``) + is **metadata**. + +- metadata: Path, relative to the root of the repository, where + :program:`dbtoyaml` and :program:`yamltodb` place or expect the YAML + specification files for the database objects when the + `--multiple-files` option is used. The default value (defined + in the system ``config.yaml``) is **metadata**. + +- path: Absolute path to the root of the repository. This should + normally be specified in a user configuration file, or in a file + given with the :option:`--config` option. If not specified, this + defaults to the current working directory from which the utility is + run. diff --git a/docs/constraint.rst b/docs/constraint.rst new file mode 100644 index 0000000..8e19147 --- /dev/null +++ b/docs/constraint.rst @@ -0,0 +1,109 @@ +Constraints +=========== + +.. module:: pyrseas.dbobject.constraint + +The :mod:`constraint` module defines six classes: :class:`Constraint` +derived from :class:`DbSchemaObject`, classes +:class:`CheckConstraint`, :class:`PrimaryKey`, :class:`ForeignKey` and +:class:`UniqueConstraint` derived from :class:`Constraint`, and +:class:`ConstraintDict` derived from :class:`DbObjectDict`. + +Constraint +---------- + +Class :class:`Constraint` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a constraint +on a database table. Its :attr:`keylist` attributes are the schema +name, the table name and the constraint name. + +.. autoclass:: Constraint + +.. automethod:: Constraint.key_columns + +.. automethod:: Constraint.add + +.. automethod:: Constraint.drop + +.. automethod:: Constraint.comment + +Check Constraint +---------------- + +:class:`CheckConstraint` is derived from :class:`Constraint` and represents +a CHECK constraint. + +.. autoclass:: CheckConstraint + +.. automethod:: CheckConstraint.to_map + +.. automethod:: CheckConstraint.add + +.. automethod:: CheckConstraint.drop + +.. automethod:: CheckConstraint.alter + +Primary Key +----------- + +:class:`PrimaryKey` is derived from :class:`Constraint` and represents +a primary key constraint. + +.. autoclass:: PrimaryKey + +.. automethod:: PrimaryKey.to_map + +.. automethod:: PrimaryKey.alter + +Foreign Key +----------- + +:class:`ForeignKey` is derived from :class:`Constraint` and represents +a foreign key constraint. + +The following shows a foreign key segment of a map returned by +:meth:`to_map` and expected as argument by +:meth:`ConstraintDict.from_map` exemplifying various possibilities:: + + {'t1_fgn_key1': + { + 'columns': ['c2', 'c3'], + 'on_delete': 'restrict', + 'on_update': 'set null', + 'references': + {'columns': ['pc2', 'pc1'], 'schema': 's1', 'table': 't2'} + } + } + +.. autoclass:: ForeignKey + +.. automethod:: ForeignKey.ref_columns + +.. automethod:: ForeignKey.to_map + +.. automethod:: ForeignKey.add + +.. automethod:: ForeignKey.alter + +Unique Constraint +----------------- + +:class:`UniqueConstraint` is derived from :class:`Constraint` and +represents a UNIQUE, non-primary key constraint. + +.. autoclass:: UniqueConstraint + +.. automethod:: UniqueConstraint.to_map + +.. automethod:: UniqueConstraint.alter + +Constraint Dictionary +--------------------- + +Class :class:`ConstraintDict` is a dictionary derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of constraints in a database. + +.. autoclass:: ConstraintDict + +.. automethod:: ConstraintDict.from_map diff --git a/docs/conversion.rst b/docs/conversion.rst new file mode 100644 index 0000000..1cc4f56 --- /dev/null +++ b/docs/conversion.rst @@ -0,0 +1,33 @@ +Conversions +=========== + +.. module:: pyrseas.dbobject.conversion + +The :mod:`conversion` module defines two classes, :class:`Conversion` +and :class:`ConversionDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Conversion +---------- + +:class:`Conversion` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +conversion between character set encodings +`_. + +.. autoclass:: Conversion + +.. automethod:: Conversion.to_map + +.. automethod:: Conversion.create + +Conversion Dictionary +--------------------- + +:class:`ConversionDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of conversions in a database. + +.. autoclass:: ConversionDict + +.. automethod:: ConversionDict.from_map diff --git a/docs/database.rst b/docs/database.rst new file mode 100644 index 0000000..0ca813c --- /dev/null +++ b/docs/database.rst @@ -0,0 +1,90 @@ +Databases +========= + +.. module:: pyrseas.database + +The :mod:`database` module defines class :class:`Database`. + +Database +-------- + +A :class:`Database` can be viewed as a tree of database objects. The +tree may have one or two main branches. A tree with one main branch +is used by :program:`dbtoyaml` to hold the representation of the +database, as read from the Postgres catalogs. :program:`yamltodb` +uses a second main branch to hold the representation as read from the +YAML input specification. + +Each main branch consists of multiple subtrees for different kinds of +objects. For example, the Schemas (Postgres namespaces) subtree has +all the Postgres schema objects, the Procedures subtree has all the +Postgres functions and aggregates. The objects in the subtrees are +connected in implicit or explicit manners to related objects. For +example, the objects in the ``schema public`` are implicitly +accessible from the corresponding :class:`Schema` object because they +all share ``public`` as the first part of their internal key (see +:meth:`DbObject.key`). As another example, a table has explicit +links to constraints and indexes defined on it. + +A :class:`Database` is initialized from a +:class:`~pyrseas.database.CatDbConnection` object (a specialized class +derived from :class:`~pyrseas.lib.dbconn.DbConnection`). It consists of +one or two :class:`Dicts` (the main branches in the above +discussion). A :class:`Dicts` object holds various dictionary objects +derived from :class:`~pyrseas.dbobject.DbObjectDict`, e.g., +:class:`~pyrseas.dbobject.schema.SchemaDict`, +:class:`~pyrseas.dbobject.table.ClassDict`, and +:class:`~pyrseas.dbobject.column.ColumnDict`. The key for each +dictionary is a Python tuple (or a single value in the case of +:class:`SchemaDict` and other non-schema objects). For example, the +:class:`~pyrseas.dbobject.table.ClassDict` dictionary is indexed by +(`schema name`, `table name`)--in this context `table name` may +actually be a `sequence name`, a `view name` or a `materialized view +name`. In addition, object instances in each dictionary are linked to +related objects in other dictionaries, e.g., columns are linked to the +tables where they belong. + +The :attr:`db` :class:`Dicts` object --always present-- instantiates +the database schemas, including their tables and other objects, by +querying the system catalogs. The :attr:`ndb` :class:`Dicts` object +instantiates the schemas based on the :obj:`input_map` supplied to the +:meth:`diff_map` method. + +The :meth:`to_map` method returns and the :meth:`diff_map` method +takes as input, a Python dictionary (equivalent to a YAML or JSON +object) as shown below. It uses 'schema `schema_name`' as the key for +each schema. The value corresponding to each 'schema `schema_name`' is +another dictionary using 'sequences', 'tables', etc., as keys and more +dictionaries as values. For example:: + + {'schema public': + {'sequence seq1': { ... }, + 'sequence seq2': { ... }, + 'table t1': { ... }, + 'table t2': { ... }, + 'table t3': { ... }, + 'view v1': { ... } + }, + 'schema s1': { ... }, + 'schema s2': { ... } + } + +Refer to :class:`~pyrseas.dbobject.table.Sequence`, +:class:`~pyrseas.dbobject.table.Table` and +:class:`~pyrseas.dbobject.table.View` for details on the lower level +dictionaries. + +.. autoclass:: Database + +Methods :meth:`from_catalog` and :meth:`from_map` are for internal +use. Methods :meth:`to_map` and :meth:`diff_map` are the external API. + +.. automethod:: Database.from_catalog + +.. automethod:: Database.from_map + +.. automethod:: Database.map_from_dir + +.. automethod:: Database.to_map + +.. automethod:: Database.diff_map diff --git a/docs/dbaugment.rst b/docs/dbaugment.rst new file mode 100644 index 0000000..701ddc3 --- /dev/null +++ b/docs/dbaugment.rst @@ -0,0 +1,108 @@ +dbaugment - Augment a database +============================== + +Name +---- + +dbaugment -- Augment a Postgres database in predefined ways + +Synopsis +-------- + +:: + + dbaugment [option...] dbname [spec] + +Description +----------- + +:program:`dbaugment` is a utility for augmenting a Postgres database +with various standard attributes and procedures, such as automatically +maintained audit columns. The augmentations are defined in a +YAML-formatted ``spec`` file. + +The following is an example of a specification file:: + + augmenter: + columns: + modified_date: + not_null: true + type: date + schema public: + table t1: + audit_columns: default + table t3: + audit_columns: modified_only + +The specification file lists each schema, and within it, each table to +be augmented. Under each table the following values are currently +recognized: + + - audit_columns: This indicates that audit trail columns are to be + added to the table, e.g., a timestamp column recording when a row + was last modified. + +The first section of the specification file, under the ``augmenter`` +header, lists configuration information. This is in addition to the +built-in configuration objects (see :ref:`predef-aug`). + +:program:`dbaugment` first reads the database catalogs. It also +initializes itself from predefined configuration information. +:program:`dbaugment` then reads the specification file, which may +include additional configuration objects, and outputs a YAML file, +including the existing catalog information together with the desired +enhancements. The YAML file is suitable for input to +:program:`yamltodb` to generate the SQL statements to implement the +changes. + +Options +------- + +:program:`dbaugment` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +**dbname** + + Specifies the name of the database whose schema is to augmented. + +**spec** + + Location of the file with the augmenter specifications. If this + is omitted, the specification is read from the program's standard + input. + +Examples +-------- + +To augment a database called ``moviesdb`` according to the +specifications in the file ``movies.yaml``:: + + dbaugment moviesdb movies.yaml + +To add a column named ``updated`` to table ``public.film`` to hold the +date and time each row was inserted or updated, create a YAML +specification file, say ``film.yaml`` as follows:: + + augmenter: + columns: + modified_timestamp: + name: updated + schema public: + table film: + audit_columns: modified_only + +The first four lines configure the predefined ``modified_timestamp`` +audit column to use the name ``updated`` instead. The last three +lines direct ``dbaugment`` to apply the predefined ``modified_only`` +audit column to the ``film`` table. + +Then run the following command to generate the resulting database +specification, alter the table and create the needed trigger and +function. + + dbaugment moviesdb film.yaml | yamltodb moviesdb -u + +See Also +-------- + + :ref:`predef-aug` diff --git a/docs/dbobject.rst b/docs/dbobject.rst new file mode 100644 index 0000000..db126f3 --- /dev/null +++ b/docs/dbobject.rst @@ -0,0 +1,101 @@ +Database Objects +================ + +.. module:: pyrseas.dbobject + +The :mod:`dbobject` module defines two low-level classes and an +intermediate class. Most Pyrseas classes are derived from either +:class:`DbObject` or :class:`DbObjectDict`. + +Database Object +--------------- + +A :class:`DbObject` represents a database object such as a schema, +table, or column, defined in a Postgres `system catalog +`_. It is +initialized from a dictionary of attributes. Derived classes should +define a :attr:`keylist` that is a list of attribute names that +uniquely identify each object instance within the database. + +.. autoclass:: DbObject + +.. autoattribute:: DbObject.objtype + +.. autoattribute:: DbObject.keylist + +.. automethod:: DbObject.key + +The following methods are generally used to map objects for external +output: + +.. automethod:: DbObject.extern_key + +.. automethod:: DbObject.query + +.. automethod:: DbObject.extern_filename + +.. automethod:: DbObject.identifier + +.. automethod:: DbObject.to_map + +.. automethod:: DbObject.map_privs + +The following methods generate SQL statements from the object +properties and sometimes from a second object: + +.. automethod:: DbObject.comment + +.. automethod:: DbObject.alter_owner + +.. automethod:: DbObject.drop + +.. automethod:: DbObject.rename + +.. automethod:: DbObject.alter + +.. automethod:: DbObject.diff_privileges + +.. automethod:: DbObject.diff_description + + +Database Object Dictionary +-------------------------- + +A :class:`DbObjectDict` represents a collection of :class:`DbObject`'s +and is derived from the Python built-in type :class:`dict`. If a +:class:`~pyrseas.lib.dbconn.DbConnection` object is used for +initialization, an internal method is called to initialize the +dictionary from the database catalogs. The :class:`DbObjectDict` +:meth:`fetch` method fetches all objects using the `query` method +defined by derived classes. Derived classes should also define a +:attr:`cls` attribute for the associated :class:`DbObject` class, +e.g., :class:`~pyrseas.schema.SchemaDict` sets :attr:`cls` to +:class:`~pyrseas.schema.Schema`. + +.. autoclass:: DbObjectDict + +.. autoattribute:: DbObjectDict.cls + +.. automethod:: DbObjectDict.to_map + +.. automethod:: DbObjectDict.fetch + + +Schema Object +------------- + +A :class:`DbSchemaObject` is derived from :class:`DbObject`. It is +used as a base class for objects owned by a schema and to define +certain common methods. This is different from the +:class:`~pyrseas.schema.Schema` that represents the schema (Postgres +namespace) itself. + +.. autoclass:: DbSchemaObject + +.. automethod:: DbSchemaObject.identifier + +.. automethod:: DbSchemaObject.qualname + +.. automethod:: DbSchemaObject.unqualify + +.. automethod:: DbSchemaObject.rename diff --git a/docs/dbtoyaml.rst b/docs/dbtoyaml.rst new file mode 100644 index 0000000..9212996 --- /dev/null +++ b/docs/dbtoyaml.rst @@ -0,0 +1,293 @@ +dbtoyaml - Database to YAML +=========================== + +Name +---- + +dbtoyaml -- extract the schema of a Postgres database in YAML format + +Synopsis +-------- + +:: + + dbtoyaml [option...] dbname + +Description +----------- + +:program:`dbtoyaml` is a utility for extracting the schema of a +Postgres database to a `YAML `_ formatted +specification. By default, the specification is output as a single +output stream, which can be redirected or explicitly sent to a file. +As an alternative, the ``--multiple-files`` option allows you to break +down the specification into multiple files, in general, one for each +object (see `Multiple File Output`_). + +Note that `JSON `_ is an official +subset of YAML version 1.2, so the :program:`dbtoyaml` output should +also be compatible with JSON tools. + +A sample of the output format is as follows:: + + schema public: + owner: postgres + privileges: + - postgres: + - all + - PUBLIC: + - all + table t1: + check_constraints: + t1_c2_check: + columns: + - c2 + expression: (c2 > 123) + columns: + - c1: + not_null: true + type: integer + - c2: + type: smallint + - c3: + default: 'false' + type: boolean + - c4: + type: text + foreign_keys: + t1_c2_fkey: + columns: + - c2 + references: + columns: + - c21 + schema: s1 + table: t2 + owner: alice + primary_key: + t1_pkey: + columns: + - c1 + schema s1: + owner: bob + privileges: + - bob: + - all + - alice: + - all + table t2: + columns: + - c21: + not_null: true + type: integer + - c22: + type: character varying(16) + owner: bob + primary_key: + t2_pkey: + columns: + - c21 + privileges: + - bob: + - all + - PUBLIC: + - select + - alice: + - insert: + grantable: true + - delete: + grantable: true + - update: + grantable: true + - carol: + grantor: alice + privs: + - insert + + +The above should be mostly self-explanatory. The example database has +two tables, named ``t1`` and ``t2``, the first --owned by user +'alice'-- in the ``public`` schema and the second --owned by user +'bob'-- in a schema named ``s1`` (also owned by 'bob'). +The ``columns:`` specifications directly under each table list each +column in that table, in the same order as shown by Postgres. The +specifications ``primary_key:``, ``foreign_keys:`` and +``check_constraints:`` define PRIMARY KEY, FOREIGN KEY and CHECK +constraints for a given table. Additional specifications (not shown) +define unique constraints and indexes. + +User 'bob' has granted all privileges to 'alice' on the ``s1`` schema. +On table ``t2``, he also granted SELECT to PUBLIC; INSERT, UPDATE and +DELETE to 'alice' with GRANT OPTION; and she has in turn granted +INSERT to user 'carol'. + +:program:`dbtoyaml` currently supports extracting information about +nearly all types of Postgres database objects. See :ref:`api-ref` +for a list of supported objects. + +The behavior and options of ``dbtoyaml`` are patterned after the +`pg_dump utility +`_ +since it is most analogous to using ``pg_dump --schema-only``. + +Multiple File Output +-------------------- + +.. program:: dbtoyaml + +The :option:`--multiple-files` option breaks down the output into +multiple files under a given root directory. The root is created if +it does not exist. The root directory name defaults to ``metadata`` +in the system configuration file. The location of the root directory +defaults to the configuration item ``repository.path`` or can be +specified using the `--repository` option (see :doc:`config` +and :doc:`cmdargs` for further details). + +The first level contains ``schema.`` subdirectories, +``schema..yaml`` files and ``..yaml`` files, +where ```` is the name of the corresponding objects and +```` is the type of top-level (non-schema) object. Note that +non-schema refers to Postgres extensions, casts, languages or +foreign data wrappers. + +The second level, i.e., the ``schema.`` subdirectories contain +``..yaml`` files for each object in the particular +schema (but see below for caveats). + +Object Name Conflicts +~~~~~~~~~~~~~~~~~~~~~ + +The names of Postgres objects can include characters that are not +allowed in filesystem object names. The most common example is the +division operator ('/'), but even table names can include +non-alphanumeric characters, if the identifiers are quoted. + +In addition, one can define two or more objects with the same base +name, e.g., function ``foo(integer)`` and function ``foo(text)``, or a +table named ``"My Table"`` and another named ``"my table"`` or +``"MY TABLE"``. On certain operating systems, i.e., Windows, it is not +possible to create two files in the same directory that differ only in +the case of their characters. + +In order to deal with the aforementioned issues, ``dbtoyaml`` places +certain objects in common files and transforms object identifiers so +that they are suitable for use in files and directories. For example, +the information for all user-defined casts are written to the file +``cast.yaml`` in the root directory. Functions with the same name but +different arguments are written to a single file, e.g., +``function.foo.yaml`` in the first example above. Identifiers are +also converted to all lowercase, non-alphanumeric characters +(excluding underscore) are converted to underscores and, by default, +schema object names are truncated to 32 characters. + +If two object names, thus transformed, map to the same string, then +the objects' information is written to the same file, e.g., +``table.my_table.yaml`` in the second example above. If you prefer to +change the default truncation length, please define the environment +variable ``PYRSEAS_MAX_IDENT_LEN`` to some integer value (up to 63). + +Version Control and Dropped Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is expected that the output of ``dbtoyaml --multiple-files`` will +be placed under version control. Further invocations should then +update the files in the same directory tree. However, if an object is +dropped from the database ``dbtoyaml`` would normally only output +files for new or changed objects--and thus keep the dropped object +file under version control. To deal with dropped objects, ``dbtoyaml +-m`` outputs a special YAML "index" file, named +``database..yaml`` in the root directory. When ``dbtoyaml +-m`` is run a second time, it looks for this "index" file and if +found, proceeds to delete the previous run's ``.yaml`` files before +outputting new ones. + +Options +------- + +:program:`dbtoyaml` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +dbname + + Specifies the name of the database whose schema is to be extracted. + +.. cmdoption:: -m, --multiple-files + + Extracts the schema to a two-level directory tree. See `Multiple + File Output`_ above. + +.. cmdoption:: -n + --schema + + Extracts only a schema matching `schema`. By default, all schemas + are extracted. Multiple schemas can be extracted by using multiple + ``-n`` switches. Note that normally all objects that belong to the + schema are extracted as well, unless excluded otherwise. + +.. cmdoption:: -N + --exclude-schema + + Does not extract schema matching `schema`. This can be given more + than once to exclude several schemas. + +.. cmdoption:: -O, --no-owner + + Do not output object ownership information. By default, as seen + in the sample output above, database objects (schemas, tables, + etc.) that can be owned by some user, are shown with an "owner: + *username*" element. The :option:`-O` switch suppresses all those + lines. + + NOTE: If you specify `--no-owner`, you will most likely also want + to specify :option:`--no-privileges`. If the former is used + without the latter the resulting YAML output will have privilege + information without user data, which will cause errors if the YAML + is then fed to :doc:`yamltodb`. + +.. cmdoption:: -t + --table
+ + Extract only tables matching `table`. Multiple tables can be + extracted by using multiple :option:`-t` switches. Note that + selecting a table may cause other objects, such as an owned + sequence, to be extracted as well + +.. cmdoption:: -T
+ --exclude-table
+ + Do not extract tables matching `table`. Multiple tables can be + excluded by using multiple :option:`-T` switches. + +.. cmdoption:: -x, --no-privileges + + Do not output access privilege information. By default, as seen + in the sample output above, if specific GRANTs have been issued on + various objects (schemas, tables, etc.), the privileges are shown + under each object. The :option:`-x` switch suppresses all those + lines. + + See also the NOTE under :option:`--no-owner`. + +Examples +-------- + +To extract a database called ``moviesdb`` into a file:: + + dbtoyaml moviesdb > moviesdb.yaml + +To extract only the schema named ``store``:: + + dbtoyaml --schema=store moviesdb > moviesdb.yaml + +To extract the tables named ``film`` and ``genre``:: + + dbtoyaml -t film -t genre moviesdb -o moviesdb.yaml + +To extract objects, to standard output, except those in schemas +``product`` and ``store``:: + + dbtoyaml -N product -N store moviesdb + +To extract objects to a directory under version control:: + + dbtoyaml moviesdb -m movies/dbspec diff --git a/docs/devel.rst b/docs/devel.rst new file mode 100644 index 0000000..2b26afb --- /dev/null +++ b/docs/devel.rst @@ -0,0 +1,97 @@ +.. _development: + +Development +=========== + +The following details the tools needed to contribute to the +development of Pyrseas. If you have any doubts or questions, please +open an issue on GitHub (https://github.com/perseas/Pyrseas/issues). +In addition, see *Version Control* below on how to set up a GitHub +account to participate in development. + +Requirements +------------ + +- Git + +- Python + +- Postgres + +- Psycopg3 + +- PyYAML + +- Tox + +Version Control +--------------- + +Pyrseas uses `Git `_ to control changes to its +source code. As mentioned under :ref:`download`, the master Git +`repository `_ is located at GitHub. + +To install Git, either `download and install +`_ the latest stable release for your +platform or follow the `Pro Git` `installation instructions +`_. For +most Linux users, ``apt-get`` or ``yum`` (depending on Linux flavor) +will be the simplest means to install the ``git-core`` package. For +Windows, downloading the installer and selecting ``Git Bash`` gives +you not only Git but a Bash shell, which is handy if you're coming +from a Linux/Unix background. + +Once Git is installed, change to a suitable directory and clone the +master repository:: + + git clone https://github.com/perseas/Pyrseas.git + +To be able to create a fork on GitHub, open an issue or participate in +Pyrseas development, you'll first have to `create a GitHub account +`_. + +Programming Language +-------------------- + +To contribute to Pyrseas, you need a version of `Python +`_. You can develop using Python 3.7 or higher. + +If Python is not already available on your machine, either `download +and install one or both `_ of the +production releases for your platform, follow the applicable +installation instructions given in `The Hitchhiker’s Guide to Python! +`_ or install it from your +platform's package management system. + +Database Installation +--------------------- + +To participate in Pyrseas development, you'll also need one or more +installations of `Postgres `_, versions +13, 12, 11 or 10. If you only have limited space, it is +preferable to install one of the latest two versions. + +The versions can be obtained as binary packages or installers from the +`Postgres.org website `_. The +site also includes instructions for installing from package management +systems or building it from source. + +To access Postgres from Python, you have to install the `Psycopg +`_ version 3 adapter. You can either follow the +instructions in `Psycopg's site +`_, or install it from +your package management system. + +Other Libraries and Tools +------------------------- + +The ``dbtoyaml`` and ``yamltodb`` utilities use the `PyYAML +`_ library. You can install it from +the PyYAML site, or possibly from your package management system. For +Windows 64-bit, please read the note under :ref:`installer`. + +To easily run the Pyrseas tests against various Python/Postgres +version combinations, you will need `pytest +`_ and `Tox +`_. Please refer to +:ref:`testing` for more information. diff --git a/docs/eventtrig.rst b/docs/eventtrig.rst new file mode 100644 index 0000000..71006c8 --- /dev/null +++ b/docs/eventtrig.rst @@ -0,0 +1,33 @@ +Event Triggers +============== + +.. module:: pyrseas.dbobject.eventtrig + +The :mod:`eventtrig` module defines two classes, :class:`EventTrigger` and +:class:`EventTriggerDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Event Trigger +-------------- + +:class:`EventTrigger` is derived from +:class:`~pyrseas.dbobject.DbObject` and represents an `event trigger +`_ +available from Postgres 9.3 onwards. + +.. autoclass:: EventTrigger + +.. automethod:: EventTrigger.to_map + +.. automethod:: EventTrigger.create + +Event Trigger Dictionary +------------------------ + +:class:`EventTriggerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of event triggers in a database. + +.. autoclass:: EventTriggerDict + +.. automethod:: EventTriggerDict.from_map diff --git a/docs/extension.rst b/docs/extension.rst new file mode 100644 index 0000000..6f5b3c3 --- /dev/null +++ b/docs/extension.rst @@ -0,0 +1,33 @@ +Extensions +========== + +.. module:: pyrseas.dbobject.extension + +The :mod:`extension` module defines two classes, :class:`Extension` +and :class:`ExtensionDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Extension +--------- + +:class:`Extension` is derived from +:class:`~pyrseas.dbobject.DbObject` and represents a `Postgres +extension +`_. + +.. autoclass:: Extension + +.. automethod:: Extension.create + +.. automethod:: Extension.alter + +Extension Dictionary +-------------------- + +:class:`ExtensionDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of extensions in a database. + +.. autoclass:: ExtensionDict + +.. automethod:: ExtensionDict.from_map diff --git a/docs/foreign.rst b/docs/foreign.rst new file mode 100644 index 0000000..d01792c --- /dev/null +++ b/docs/foreign.rst @@ -0,0 +1,142 @@ +Foreign Data Objects +==================== + +.. module:: pyrseas.dbobject.foreign + +The :mod:`foreign` module defines nine classes related to Postgres +foreign data wrappers (FDWs), namely: :class:`DbObjectWithOptions` +derived from :class:`DbObject`, classes :class:`ForeignDataWrapper`, +:class:`ForeignServer` and :class:`UserMapping` derived from +:class:`DbObjectWithOptions`, :class:`ForeignTable` derived from +:class:`DbObjectWithOptions` and :class:`Table`, classes +:class:`ForeignDataWrapperDict`, :class:`ForeignServerDict` and +:class:`UserMappingDict` derived from :class:`DbObjectDict`, and +:class:`ForeignTableDict` derived from :class:`ClassDict`. + +Database Object With Options +---------------------------- + +:class:`DbObjectWithOptions` is derived from +:class:`~pyrseas.dbobject.DbObject`. It is a helper class for dealing +with the OPTIONS clauses common to the foreign data objects. + +.. autoclass:: DbObjectWithOptions + +.. automethod:: DbObjectWithOptions.to_map + +.. automethod:: DbObjectWithOptions.options_clause + +.. automethod:: DbObjectWithOptions.diff_options + +.. automethod:: DbObjectWithOptions.alter + +Foreign Data Wrapper +-------------------- + +:class:`ForeignDataWrapper` is derived from `DbObjectWithOptions` and +represents a `Postgres foreign data wrapper +`_. +See also `Foreign Data +`_ +and `Writing A Foreign Data Wrapper +`_. + +.. autoclass:: ForeignDataWrapper + +.. automethod:: ForeignDataWrapper.to_map + +.. automethod:: ForeignDataWrapper.create + +Foreign Data Wrapper Dictionary +------------------------------- + +:class:`ForeignDataWrapperDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of foreign data wrappers in a database. + +.. autoclass:: ForeignDataWrapperDict + +.. automethod:: ForeignDataWrapperDict.from_map + +Foreign Server +-------------- + +:class:`ForeignServer` is derived from :class:`DbObjectWithOptions` +and represents a `Postgres foreign server +`_. + +.. autoclass:: ForeignServer + +.. automethod:: ForeignServer.identifier + +.. automethod:: ForeignServer.to_map + +.. automethod:: ForeignServer.create + +Foreign Server Dictionary +------------------------- + +:class:`ForeignServerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a Python dictionary +that represents the collection of foreign servers in a database. + +.. autoclass:: ForeignServerDict + +.. automethod:: ForeignServerDict.from_map + +.. automethod:: ForeignServerDict.to_map + +User Mapping +------------ + +:class:`UserMapping` is derived from :class:`DbObjectWithOptions` and +represents a `mapping of a Postgres user to a foreign server +`_. + +.. autoclass:: UserMapping + +.. automethod:: UserMapping.extern_key + +.. automethod:: UserMapping.identifier + +.. automethod:: UserMapping.create + +User Mapping Dictionary +----------------------- + +:class:`UserMappingDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of user mappings in a database. + +.. autoclass:: UserMappingDict + +.. automethod:: UserMappingDict.from_map + +.. automethod:: UserMappingDict.to_map + +Foreign Table +------------- + +:class:`ForeignTable` is derived from :class:`DbObjectWithOptions` and +:class:`~pyrseas.dbobject.table.Table`. It represents a `Postgres +foreign table +`_. + +.. autoclass:: ForeignTable + +.. automethod:: ForeignTable.to_map + +.. automethod:: ForeignTable.create + +.. automethod:: ForeignTable.drop + +Foreign Table Dictionary +------------------------ + +:class:`ForeignTableDict` is derived from +:class:`~pyrseas.dbobject.table.ClassDict`. It is a dictionary that +represents the collection of foreign tables in a database. + +.. autoclass:: ForeignTableDict + +.. automethod:: ForeignTableDict.from_map diff --git a/docs/function.rst b/docs/function.rst new file mode 100644 index 0000000..4b89349 --- /dev/null +++ b/docs/function.rst @@ -0,0 +1,65 @@ +Functions +========= + +.. module:: pyrseas.dbobject.function + +The :mod:`function` module defines four classes: class :class:`Proc` +derived from :class:`DbSchemaObject`, classes :class:`Function` and +:class:`Aggregate` derived from :class:`Proc`, and class +:class:`ProcDict` derived from :class:`DbObjectDict`. + +Procedure +--------- + +Class :class:`Proc` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a regular or +aggregate function. + +.. autoclass:: Proc + +.. automethod:: Proc.extern_key + +.. automethod:: Proc.identifier + +Function +-------- + +:class:`Function` is derived from :class:`Proc` and represents a +`Postgres user-defined function +`_. + + +.. autoclass:: Function + +.. automethod:: Function.to_map + +.. automethod:: Function.create + +.. automethod:: Function.alter + +.. automethod:: Function.drop + +Aggregate Function +------------------ + +:class:`Aggregate` is derived from :class:`Proc` and represents a +`Postgres user-defined aggregate function +`_. + +.. autoclass:: Aggregate + +.. automethod:: Aggregate.to_map + +.. automethod:: Aggregate.create + +Procedure Dictionary +-------------------- + +:class:`ProcDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of regular and aggregate functions in a +database. + +.. autoclass:: ProcDict + +.. automethod:: ProcDict.from_map diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4bc7ae2 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,132 @@ +Pyrseas +======= + +Pyrseas provides utilities to describe a PostgreSQL database schema as +YAML, to verify the schema against the same or a different database +and to generate SQL that will modify the schema to match the YAML +description. + +Features +-------- + +- Outputs a YAML/JSON description of a PostgreSQL database's tables + and other objects (metadata), suitable for storing in a version + control repository + +- Generates SQL statements to modify a database so that it will match + an input YAML/JSON specification + +- Generates an augmented YAML description of a PostgreSQL database + from its catalogs and an augmentation specification. + +Requirements +------------ + +- `PostgreSQL `_ 10 or higher + +- `Python `_ 3.7 or higher + +- `Psycopg3 `_ 3.1 or higher + +- `PyYAML `_ 5.3 or higher + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + overview + install + config + configitems + devel + testing + issues + predefaug +.. toctree:: + :maxdepth: 1 + + dbaugment + dbtoyaml + yamltodb + cmdargs + +.. _api-ref: + +API Reference +------------- + +Currently, the only external APIs are the class +:class:`~pyrseas.database.Database` and the methods +:meth:`~pyrseas.database.Database.to_map` and +:meth:`~pyrseas.database.Database.diff_map` of the latter. Other +classes and methods are documented mainly for developer use. + +.. toctree:: + :maxdepth: 2 + + dbobject + database + schema + +Non-schema Objects +~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + cast + eventtrig + extension + foreign + language + +Tables and Related Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + table + column + constraint + indexes + rule + +Functions, Operators and Triggers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + function + operator + operfamily + operclass + trigger + +Types and Other Schema Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + collation + conversion + textsearch + type + +Augmenter API Reference +----------------------- + +.. toctree:: + :maxdepth: 2 + + augmentdb + cfgobjects + augobjects + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/indexes.rst b/docs/indexes.rst new file mode 100644 index 0000000..ed07d62 --- /dev/null +++ b/docs/indexes.rst @@ -0,0 +1,43 @@ +Indexes +======= + +.. module:: pyrseas.dbobject.index + +The :mod:`index` module defines two classes, :class:`Index` and +:class:`IndexDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Index +----- + +Class :class:`Index` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents an index on a +database table, other than a primary key or unique constraint +index. Its :attr:`keylist` attributes are the schema name, the table +name and the index name. Note that index names are supposed to be +unique with a given schema so the table name doesn't have to be part +of the :attr:`keylist`, but has been retained to facilitate certain +operations. + +.. autoclass:: Index + +.. automethod:: Index.key_expressions + +.. automethod:: Index.to_map + +.. automethod:: Index.create + +.. automethod:: Index.alter + +.. automethod:: Index.drop + +Index Dictionary +---------------- + +Class :class:`IndexDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of indexes in a database. + +.. autoclass:: IndexDict + +.. automethod:: IndexDict.from_map diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 0000000..70ba9da --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,182 @@ +Installation +============ + +Summary +------- + +For the latest release, use:: + + pip install Pyrseas + +For development:: + + git clone git://github.com/perseas/Pyrseas.git + cd Pyrseas + python setup.py install + +Requirements +------------ + +Pyrseas provides tools for `Postgres `_, +so obviously you need **Postgres** to start with. Pyrseas has been +tested with PG 10, 11, 12 and 13 and we'll certainly keep up +with future releases. Please refer to the `Postgres download page +`_ to find a distribution for the +various Linux, Unix and Windows platforms supported. + +You will also need **Python**. Pyrseas was originally developed using +`Python ` 2 and then ported to Python 3 and +tested against versions from 3.7 through 3.9. On Linux or \*BSD, +Python may already be part of your distribution or may be available as +a package. For Windows and Mac OS please refer to the `Python +download page `_ for installers and +instructions. + +Pyrseas talks to the Postgres DBMS via the **Psycopg adapter**. +Pyrseas has been tested with `psycopg `_ +3.1. Psycopg may be available as a package on most Linux or +\*BSD distributions and can also be downloaded or installed from PyPI. +Please refer to the `Psycopg download page +`_ for more details. + +.. note:: If you install Pyrseas using ``pip`` (see below) and you + have not already installed Psycopg, e.g., when installing into a + ``virtualenv`` environment created with ``--no-site-packages``, you + may need to have installed the Postgres and Python development + packages, and a C compiler, as ``pip`` may download and attempt to + build and install psycopg before installing Pyrseas. + +The Pyrseas utilities rely on **PyYAML**, a `YAML `_ +library. This may be available as a package for your operating system +or it can be downloaded from the `Python Package Index (PyPI) +`_. + +.. _download: + +Downloading +----------- + +Pyrseas is available at the following locations: + + - `Python Package Index `_ + - `Postgres Extension Network (PGXN) `_ + - `GitHub repository `_ + +You can download the distribution from PyPI in gzip-compressed tar or +ZIP archive format, but you can download *and* install it using +``Pip``. See `Python Installer`_ below for details. + +PGXN provides a ZIP archive which you can download or you can download +*and* install using the PGXN client (see `PGXN Client`_ below). + +The GitHub repository holds the Pyrseas source code, tagged according +to the various releases, e.g., v0.9.0, and including unreleased +modifications. To access it, you need `Git `_ +which is available as a package in most OS distributions or can be +downloaded from the `Git download page +`_. You can fetch the Pyrseas sources by +issuing the following command:: + + git clone https://github.com/perseas/Pyrseas.git + +This will create a ``Pyrseas`` directory tree (you can use a different +target name by adding it to the above commands). To list available +releases, change to the subdirectory and invoke ``git tag``. To +switch to a particular release, use:: + + git checkout vn.n.n + +where *vn.n.n* is the release identifier. Use ``git checkout master`` +to revert to the main (master) branch. To fetch the latest updates, +use:: + + git pull + +Installation +------------ + +Extracting Sources +~~~~~~~~~~~~~~~~~~ + +Once you have downloaded an archive from PyPI or PGXN, you need to +extract the sources. For a gzip-compressed tar file, use:: + + tar xzf Pyrseas-n.n.n.tar.gz + +where *n.n.n* is the release version. For a ZIP archive, use:: + + unzip Pyrseas-n.n.n.zip + +Both commands above will create a directory ``Pyrseas-n.n.n`` and you +will want to ``cd`` to it before proceeding with the installation. + +Installing +~~~~~~~~~~ + +If you have superuser or similar administrative privileges, you can +install Pyrseas for access by multiple users on your system. On Linux +and other Unix-flavored systems, you can install from the extracted +``Pyrseas-n.n.n`` source directory or from the root directory of the +``git`` clone, using the following command:: + + sudo python setup.py install + +That will install the :doc:`dbtoyaml ` and :doc:`yamltodb +` utility scripts in a directory such as +``/usr/local/bin``. The library sources and bytecode files will be +placed in a ``pyrseas`` subdirectory under ``site-packages`` or +``dist-packages``, e.g., +``/usr/local/lib/python3.6/dist-packages/pyrseas``. + +On Windows, from an account with Administrator privileges, you can +use:: + + python setup.py install + +That will install the Pyrseas utilities in the ``Scripts`` folder of +your Python installation. The source and bytecode files will go in +the ``site-packages`` folder, e.g., +``C:\Python37\Lib\site-packages\pyrseas``. + +.. _installer: + +Python Installer +~~~~~~~~~~~~~~~~ + +You can also download and install Pyrseas using `pip +`_. For example, on Linux do:: + + sudo pip install Pyrseas + +If this is the first time you are installing a Python package, please +do yourself a favor and read and follow the instructions in the +"Distribute & Pip" subsection of the "Installing Python on ..." +section for your platform of the `The Hitchhiker’s Guide to Python! +`_. + +.. note:: On FreeBSD, it has been reported that it is necessary to + install the Python ``distribute`` package, prior to + installing Pyrseas with ``pip``. This may also be necessary + on other BSD variants. + +.. note:: On Windows 64-bit, it has been reported that it is necessary + to obtain unofficial versions of the ``distribute`` and + ``PyYAML`` packages, available at `University of California, + Irvine `_. For + a detailed tutorial, see `this post + `_. + +``Pip`` can also be used in a Python `virtualenv +`_ environment, in which case +you *don't* need to prefix the commands with ``sudo``. + +``Pip`` also provides the ability to uninstall Pyrseas. + +PGXN Client +~~~~~~~~~~~ + +The PGXN `client `_ (available +at PyPI) can be used to download and install Pyrseas from PGXN. Usage +is:: + + pgxn install pyrseas diff --git a/docs/issues.rst b/docs/issues.rst new file mode 100644 index 0000000..97cde21 --- /dev/null +++ b/docs/issues.rst @@ -0,0 +1,114 @@ +Known Issues +============ + +The following summarizes notable deficiencies in the current release +of the Pyrseas utilities. For further details please refer to the +discussions in the `Pyrseas issue tracker +`_. Suggestions or patches +to deal with these issues are welcome. + +Coverage of Postgres Objects +---------------------------- + +An important Pyrseas objective is to support creating, altering or +dropping nearly any Postgres object accessible through SQL, including +adding, modifying or removing any attributes or features of those +objects. At present, we believe Pyrseas covers roughly over 90% of +the Postgres object/attribute universe. Please refer to the `Feature +Matrix `_ for details. + +This is a continuing effort since Postgres keeps adding new features +in each release, such as the table PARTITIONING syntax in PG 10. We +have documented current limitations in the issue tracker, see, for +example, issues `135 `_ +and `178 `_. Please +open an issue on the tracker if you find objects or features needing +additional support. + +Object Dependencies +------------------- + +The first releases of :program:`yamltodb` used a generally fixed +traversal order when generating SQL. This caused problems with +complex dependencies between objects (e.g., views that depended on +functions that depended on types). Release 0.8 introduced a +topological sort of objects based on their dependencies. The +resulting dependency graph is now used to drive SQL generation. This +should eliminate most object dependency problems seen with the +previous architecture. However, certain issues still remain. +Specifically, if an object depends on a Postgres internally-defined +object, or on an object defined by a Postgres extension, the Pyrseas +utilities may not behave as expected (see issue `175 +`_ for additional +discussion). + +Object renaming +--------------- + +Pyrseas provides support for generating SQL statements to rename +various database objects, e.g., ALTER TABLE t1 RENAME TO t2, using an +'oldname' tag which can be added to objects that support SQL RENAME. +The tag has to be added manually to a YAML specification for yamltodb +to act on it and cannot be kept in the YAML file for subsequent runs. +This is not entirely satisfactory for storing the YAML file in a +version control system. + +Memory utilization +------------------ + +The yamltodb utility compares the existing and input metadata by +constructing parallel, in-memory representations of the database +catalogs and the input YAML specification. If the database has a +large number of objects, e.g., in the thousands of tables, the +utility's memory usage may be noticeable. + + +Multiline Strings +----------------- + +The text of function source code, view definitions or object COMMENTs +present a problem when they span multiple lines. The default YAML +output format is to enclose the entire string in double quotes, to +show newlines that are part of the text as escaped characters (i.e., +``\n``) and to break the text into lines with a +backslash-newline-indentation-backslash pattern. For example:: + + source: "\n SELECT inventory_id\n FROM inventory\n WHERE film_id =\ + \ $1\n AND store_id = $2\n AND inventory_in_stock(inventory_id);\n" + +This is not very readable, but it does allow YAML to read it back and +correctly reconstruct the original string. To improve readability, +Pyrseas 0.7 introduced special processing for these strings. By using +YAML notation, the same string is represented as follows:: + + source: |2 + + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND NOT inventory_in_stock(inventory_id); + +Note also that if your function source code has trailing spaces at the +end of lines, they would normally be represented in the original +default format. However, in the interest of readability, +:program:`dbtoyaml` will remove the trailing spaces from the text. + +Index and Partitioning Expressions +---------------------------------- + +Postgres allows users to create `indexes using expressions +`_. +A user can also mix expressions with regular columns. The Postgres +catalogs store the index information in a bespoke fashion: an array of +column numbers where a zero indicates an expression and a list of +expression trees (an internal format) for the expressions, with +additional arrays for collation information, operator classes and +index options such as ``ASC`` or ``DESC``. Although the +``pg_get_indexdef`` system catalog function can be used to obtain a +full ``CREATE INDEX`` statement, Pyrseas has chosen to specify each +column or expression separately in the YAML definitions. This has +not been satisfactory in complex cases (see for example issue `170 +`_) and is an area +requiring further attention. A similar situation exists for table +partitioning using expressions. diff --git a/docs/language.rst b/docs/language.rst new file mode 100644 index 0000000..4e4542b --- /dev/null +++ b/docs/language.rst @@ -0,0 +1,35 @@ +Procedural Languages +==================== + +.. module:: pyrseas.dbobject.language + +The :mod:`language` module defines two classes, :class:`Language` and +:class:`LanguageDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Procedural Language +------------------- + +:class:`Language` is derived from :class:`~pyrseas.dbobject.DbObject` +and represents a procedural language. + + +.. autoclass:: Language + +.. automethod:: Language.to_map + +.. automethod:: Language.drop + +.. automethod:: Language.create + +Language Dictionary +------------------- + +:class:`LanguageDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of procedural languages in a +database. Internal languages ('internal', 'c' and 'sql') are excluded. + +.. autoclass:: LanguageDict + +.. automethod:: LanguageDict.from_map diff --git a/docs/operator.rst b/docs/operator.rst new file mode 100644 index 0000000..7530daa --- /dev/null +++ b/docs/operator.rst @@ -0,0 +1,36 @@ +Operators +========= + +.. module:: pyrseas.dbobject.operator + +The :mod:`operator` module defines two classes: class +:class:`Operator` derived from :class:`DbSchemaObject` and class +:class:`OperatorDict` derived from :class:`DbObjectDict`. + +Operator +--------- + +:class:`Operator` is derived from :class:`DbSchemaObject` and +represents a `Postgres user-defined operator +`_. + +.. autoclass:: Operator + +.. automethod:: Operator.extern_key + +.. automethod:: Operator.qualname + +.. automethod:: Operator.identifier + +.. automethod:: Operator.create + +Operator Dictionary +------------------- + +:class:`OperatorDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operators in a database. + +.. autoclass:: OperatorDict + +.. automethod:: OperatorDict.from_map diff --git a/docs/operclass.rst b/docs/operclass.rst new file mode 100644 index 0000000..9a53ded --- /dev/null +++ b/docs/operclass.rst @@ -0,0 +1,36 @@ +Operator Classes +================ + +.. module:: pyrseas.dbobject.operclass + +The :mod:`operclass` module defines two classes: class +:class:`OperatorClass` derived from :class:`DbSchemaObject` and class +:class:`OperatorClassDict` derived from :class:`DbObjectDict`. + +Operator Class +-------------- + +:class:`OperatorClass` is derived from :class:`DbSchemaObject` and +represents a `Postgres operator class +`_. + +.. autoclass:: OperatorClass + +.. automethod:: OperatorClass.extern_key + +.. automethod:: OperatorClass.identifier + +.. automethod:: OperatorClass.to_map + +.. automethod:: OperatorClass.create + +Operator Class Dictionary +------------------------- + +:class:`OperatorClassDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operator classes in a database. + +.. autoclass:: OperatorClassDict + +.. automethod:: OperatorClassDict.from_map diff --git a/docs/operfamily.rst b/docs/operfamily.rst new file mode 100644 index 0000000..b2d407d --- /dev/null +++ b/docs/operfamily.rst @@ -0,0 +1,35 @@ +Operator Families +================= + +.. module:: pyrseas.dbobject.operfamily + +The :mod:`operfamily` module defines two classes: class +:class:`OperatorFamily` derived from :class:`DbSchemaObject` and class +:class:`OperatorFamilyDict` derived from :class:`DbObjectDict`. + +Operator Family +--------------- + +:class:`OperatorFamily` is derived from :class:`DbSchemaObject` and +represents a `Postgres operator family +`_, +a grouping of related operator classes. + +.. autoclass:: OperatorFamily + +.. automethod:: OperatorFamily.extern_key + +.. automethod:: OperatorFamily.identifier + +.. automethod:: OperatorFamily.create + +Operator Family Dictionary +-------------------------- + +:class:`OperatorFamilyDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operator families in a database. + +.. autoclass:: OperatorFamilyDict + +.. automethod:: OperatorFamilyDict.from_map diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..ca1b085 --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,117 @@ +.. -*- coding: utf-8 -*- + +Overview +======== + +Pyrseas provides utilities to maintain a `PostgreSQL +`_ database schema. Its purpose is to +enhance and follow through on the concepts of the `Andromeda Project +`_. + +Whereas Andromeda expects the database designer or developer to +provide a single `YAML `_ specification file of the +database to be created, Pyrseas allows the development database to be +created using the familiar SQL CREATE statements. The developer can +then run the `dbtoyaml` utility to generate the YAML specification +from the database. The spec can then be stored in any desired version +control (VCS) repository. Similarly, she can add columns or modify +tables or other objects using SQL ALTER statements and regenerate the +YAML spec with dbtoyaml. + +When ready to create or upgrade a test or production database, the +`yamltodb` utility can be used with the YAML spec as input, to generate +a script of SQL CREATE or ALTER statements to modify the database so +that it matches the input spec. + +A third tool, `dbaugment`, can be used to add custom attributes and +supporting objects to a given schema. For example, an `updated` +column can be added to various tables, together with trigger functions +to ensure the columns are automatically modified as changes are made. + +Use Cases +--------- + +The following sections discuss the main scenarios where Pyrseas +tools may be helpful. + +Version Control +--------------- + +The case for implementing a tool to facilitate version control over +SQL databases was made in a couple of blog posts: `Version +Control, Part 1: Pre-SQL +`_ +and `Version Control, Part 2: SQL Databases +`_. In +summary, SQL data definition commands are generally incompatible with +traditional version control approaches which usually require +comparisons (diffs) between revisions of source files. + +A refinement of the approach described in the aforementioned blog +posts will be of interest to users with many objects in their database +schemas, i.e., many tables, views, functions, and other more complex +objects. Instead of storing a complete database specification in a +single YAML file, by using the `--multiple-files` option to +:program:`dbtoyaml`, the specification can be broken down into files +corresponding, generally, to a single database object. This allows a +VCS **diff** facility to easily highlight database changes. Please +refer to the :doc:`dbtoyaml` and :doc:`yamltodb` utilities for further +details. + +The Pyrseas version control tools are not designed to be the ultimate +SQL database version control solution. Instead, they are aimed at +assisting two or more developers or DBAs in sharing changes to the +underlying database as they implement a database application. The +sharing can occur through a distributed or centralized VCS. The +Pyrseas tools may even be used by a single DBA in conjunction with a +distributed VCS to quickly explore alternative designs. The tools can +also help to share changes with a conventional QA team, but may +require additional controls for final releases and production +installations. + +Supplementary Actions +--------------------- + +In many instances, a database schema needs to be supplemented by +rarely-modified data kept in certain tables, e.g., a +codes-descriptions table. The data import and export features, +controlled by `datacopy` configuration parameter (see +:doc:`configitems` for details) facilitates this need. + +In other cases, DBAs may want to standardize certain additional table +columns or processing. For example, they may want to capture the user +and time of modification of a certain set of tables using a common +procedure. The :doc:`dbaugment` utility was introduced to support +these needs. + +Generating SQL by determining what changed between one schema version +and another is sometimes not sufficient. Although the change may be +as simple as adding a column to a table and adding a referential +constraint to the new column, if the tables already have data, it may +not be possible to run the SQL generated by :program:`yamltodb`. This +requires manual intervention by the DBAs or developers. The project +would like to assist with these types of changes. The blog post `The +Future of Pyrseas: Part 1 +`_ +is a first step in discussing this requirement. + +Naming +------ + +The project name comes from `Python `_, the +programming language, and `Perseas +`_ [#]_, the Greek mythological +hero who rescued Andromeda from a sea monster [#]_. It is hoped that +Pyrseas will rescue the Andromeda project . You can pronounce +Pyrseas like the hero. + + +.. rubric:: Footnotes + +.. [#] The common English name for Perseas is Perseus and the Ancient + Greek name is Perseos. However, in modern Greek Περσέας_ is the + more common spelling for the mythical hero. + +.. _Περσέας: https://en.wiktionary.org/wiki/%CE%A0%CE%B5%CF%81%CF%83%CE%AD%CE%B1%CF%82 + +.. [#] He is better known for having killed Medusa. diff --git a/docs/predefaug.rst b/docs/predefaug.rst new file mode 100644 index 0000000..1dc3456 --- /dev/null +++ b/docs/predefaug.rst @@ -0,0 +1,93 @@ +.. _predef-aug: + +Predefined Database Augmentations +================================= + +These augmentations are specified in the ``config.yaml`` configuration +file distributed with Pyrseas' :program:`dbaugment`. + +Columns +------- + +These are predefined column specifications that can be added to +tables, e.g., in various audit column combinations (see `Audit +Columns`_ below). + +- created_by_ip_address: An INET column to record the IP address which + originated the current row. + +- created_by_user: A VARCHAR(63) column to record the user, e.g., + CURRENT_USER, who created the current row. + +- created_date: A DATE column that defaults to CURRENT_DATE. + +- created_timestamp: A TIMESTAMP WITH TIME ZONE column to record the + date and time when the current row was created. + +- modified_by_ip_address: An INET column to record the IP address + which originated the last modification to the current row. + +- modified_by_user: A VARCHAR(63) column to record the user, e.g., + CURRENT_USER, who last modified the current row. + +- modified_timestamp: A TIMESTAMP WITH TIME ZONE column to record the + date and time when the current row was last modified. + +Functions +--------- + +The following are predefined trigger functions which are used to +implement various augmentations. The source for each function, +written in PL/pgSQL, is specified in a function template, named with a +``functempl_`` prefixed to the function name. + +- Audit when modified (``audit_modified``): This function provides the + CURRENT_TIMESTAMP value for audit columns. + +- Default audit (``audit_default``): This function provides the + CURRENT_USER and CURRENT_TIMESTAMP for audit columns. + +- Full audit (``audit_full``): For SQL INSERTs, this function provides + values for the user who created the row, the CURRENT_TIMESTAMP and + the IP address for both the ``created_`` and ``modified_`` audit + columns. For UPDATEs, it retains the existing values in the + ``created_`` columns and supplies current values for the + ``modified_`` columns. + +In addition, the following helper functions are defined in schema +``pyrseas``: + +- get_session_variable +- set_session_variable + +A variant of ``get_session_variable`` is invoked by the ``audit_full`` +function to retrieve the actual (logged-on) user and IP address. In +web applications, the user that connects to the database is typically +the system user running the web server, rather than the application +(logged on) user. The application can invoke the +``pyrseas.set_session_variable`` function to supply the application +user and IP address so that the audit trail will reflect the +application context corrrectly. + +Audit Columns +------------- + +These are predefined combinations of columns to be added to tables to +record audit trail information. They may also include triggers to be +invoked to maintain the column values. + +- created_date_only: This is the simplest audit trail that adds a + ``created_date`` column which defaults to the CURRENT_DATE. + +- modified_only: This is another simple audit trail. It adds a + ``modified_timestamp`` column which is supplied by a trigger named + `table_name`\_20_audit_modified_only. + +- default: This is the default for audit columns. It adds the columns + ``modified_by_user`` and ``modified_timestamp`` and a trigger named + `table_name`\_20_audit_default to fill in the columns. + +- full: This is the most extensive audit trail combination. It adds + ``created_`` and ``modified_`` columns for user, IP address and + timestamp. It also adds a trigger named + `table_name`\_20_audit_full. diff --git a/docs/rule.rst b/docs/rule.rst new file mode 100644 index 0000000..aff3dd3 --- /dev/null +++ b/docs/rule.rst @@ -0,0 +1,35 @@ +Rules +===== + +.. module:: pyrseas.dbobject.rule + +The :mod:`rule` module defines two classes, :class:`Rule` and +:class:`RuleDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Rule +---- + +:class:`Rule` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +rewrite rule +`_. + +.. autoclass:: Rule + +.. automethod:: Rule.identifier + +.. automethod:: Rule.to_map + +.. automethod:: Rule.create + +Rule Dictionary +--------------- + +:class:`RuleDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of rewrite rules in a database. + +.. autoclass:: RuleDict + +.. automethod:: RuleDict.from_map diff --git a/docs/schema.rst b/docs/schema.rst new file mode 100644 index 0000000..f99d26b --- /dev/null +++ b/docs/schema.rst @@ -0,0 +1,57 @@ +Schemas +======= + +.. module:: pyrseas.dbobject.schema + +The :mod:`schema` module defines two classes, :class:`Schema` and +:class:`SchemaDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Schema +------ + +:class:`Schema` is derived from :class:`~pyrseas.dbobject.DbObject` +and represents a database schema or Postgres namespace, i.e., a +collection of tables and other objects. The 'public' schema is +currently treated specially as in most contexts an unqualified object +is assumed to be part of it, e.g., table "t" is usually shorthand for +table "public.t." The 'pyrseas' schema, if present, is excluded as it +is only intended for use by :program:`dbaugment` or other Pyrseas +internal purposes. + +.. autoclass:: Schema + +.. automethod:: Schema.extern_dir + +.. automethod:: Schema.to_map + +.. automethod:: Schema.create + +.. automethod:: Schema.drop + +.. automethod:: Schema.data_import + +Schema Dictionary +----------------- + +:class:`SchemaDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of schemas in a database. Certain internal +schemas (information_schema, pg_catalog, etc.) owned by the 'postgres' +user are excluded. + +.. autoclass:: SchemaDict + +Method :meth:`from_map` is called from :class:`Database` +:meth:`~pyrseas.database.Database.from_map` to start a recursive +interpretation of the input map. The :obj:`inmap` argument is the same +as input to the :meth:`~pyrseas.database.Database.diff_map` method of +:class:`Database`. The :obj:`newdb` argument is the holder of +:class:`~pyrseas.dbobject.DbObjectDict`-derived dictionaries which is +filled in as the recursive interpretation proceeds. + +.. automethod:: SchemaDict.from_map + +.. automethod:: SchemaDict.to_map + +.. automethod:: SchemaDict.data_import diff --git a/docs/table.rst b/docs/table.rst new file mode 100644 index 0000000..fc7cd37 --- /dev/null +++ b/docs/table.rst @@ -0,0 +1,178 @@ +Tables, Views and Sequences +=========================== + +.. module:: pyrseas.dbobject.table + +The :mod:`table` and :mod:`view` modules define six classes, +:class:`DbClass` derived from :class:`DbSchemaObject`, classes +:class:`Sequence`, :class:`Table` and :class:`View` derived from +:class:`DbClass`, :class:`MaterializedView` derived from +:class:`View`, and :class:`ClassDict`, derived from +:class:`DbObjectDict`. + +Database Class +-------------- + +Class :class:`DbClass` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a table, view +or sequence as defined in the Postgres `pg_class` catalog. + +.. autoclass:: DbClass + +Sequence +-------- + +Class :class:`Sequence` is derived from :class:`DbClass` and +represents a `sequence generator +`_. +Its :attr:`keylist` attributes are the schema name and the sequence +name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has the following structure:: + + {'sequence seq1': + {'cache_value': 1, + 'data_type': 'integer', + 'increment_by': 1, + 'max_value': None, + 'min_value': None, + 'owner_column': 'c1', + 'owner_table': 't1', + 'start_value': 1 + } + } + +.. autoclass:: Sequence + +.. automethod:: Sequence.get_attrs + +.. automethod:: Sequence.get_dependent_table + +.. automethod:: Sequence.to_map + +.. automethod:: Sequence.create + +.. automethod:: Sequence.add_owner + +.. automethod:: Sequence.alter + +.. automethod:: Sequence.drop + +Table +----- + +Class :class:`Table` is derived from :class:`DbClass` and represents a +database table. Its :attr:`keylist` attributes are the schema name and +the table name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has a structure similar to the following:: + + {'table t1': + {'columns': + [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'smallint'}}, + {'c4': {'type': 'date', 'default': 'now()'}} + ], + 'description': "this is the comment for table t1", + 'primary_key': + {'t1_prim_key': + {'columns': ['c1', 'c2']} + }, + 'foreign_keys': + {'t1_fgn_key1': + {'columns': ['c2', 'c3'], + 'references': + {'table': 't2', 'columns': ['pc2', 'pc1']} + }, + 't1_fgn_key2': + {'columns': ['c2'], + 'references': {'table': 't3', 'columns': ['qc1']} + } + }, + 'unique_constraints': {...}, + 'indexes': {...} + } + } + +The values for :obj:`unique_constraints` and :obj:`indexes` follow a +pattern similar to :obj:`primary_key`, but there can be more than one +such specification. + +.. autoclass:: Table + +.. automethod:: Table.column_names + +.. automethod:: Table.to_map + +.. automethod:: Table.create + +.. automethod:: Table.drop + +.. automethod:: Table.diff_options + +.. automethod:: Table.alter + +.. automethod:: Table.alter_drop_columns + +.. automethod:: Table.data_export + +.. automethod:: Table.data_import + +Class Dictionary +---------------- + +Class :class:`ClassDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of tables, views and sequences in a database. + +.. autoclass:: ClassDict + +.. automethod:: ClassDict.from_map + + +.. module:: pyrseas.dbobject.view + +View +---- + +Class :class:`View` is derived from :class:`DbClass` and represents a +database view. Its :attr:`keylist` attributes are the schema name and +the view name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has a structure similar to the following:: + + {'view v1': + {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'date'}}], + 'definition': " SELECT ...;", + 'description': "this is the comment for view v1" + } + } + + +.. autoclass:: View + +.. automethod:: View.to_map + +.. automethod:: View.create + +.. automethod:: View.alter + +Materialized View +----------------- + +Class :class:`MaterializedView` is derived from :class:`View` and +represents a `materialized view +`_. Its +:attr:`keylist` attributes are the schema name and the view name. + +.. autoclass:: MaterializedView + +.. automethod:: MaterializedView.to_map + +.. automethod:: MaterializedView.create diff --git a/docs/testing.rst b/docs/testing.rst new file mode 100644 index 0000000..5b56903 --- /dev/null +++ b/docs/testing.rst @@ -0,0 +1,175 @@ +.. _testing: + +Testing +======= + +The majority of Pyrseas' capabilities are exercised and verified via +unit tests written using `pytest +`_. The tests can be run from the +command line by most users, e.g., + +:: + + pytest-3 tests/dbobject/test_table.py + pytest-3 tests/dbobject/test_trigger.py -k test_create_trigger + pytest-3 tests/functional + +The first ``pytest-3`` command above runs all tests related to tables, +mapping, creating, dropping, etc. The second one executes a single +test to generate SQL to create a trigger. The third runs all the +functional tests. Please review the `pytest documentation +`_ for further options. + +Environment Variables +--------------------- + +By default, the tests use a Postgres database named ``pyrseas_testdb`` +which is created if it doesn't already exist. The tests are run as the +logged in user, using the ``USER`` Unix/Linux environment variable (or +``USERNAME`` under Windows). They access Postgres on the local host +using the default port number (5432). + +The following four environment variables can be used to change the +defaults described above: + + - PYRSEAS_TEST_DB + - PYRSEAS_TEST_USER + - PYRSEAS_TEST_HOST + - PYRSEAS_TEST_PORT + +Restrictions +------------ + +Unless the test database exists and the user running the tests has +access to it, the user role will need CREATEDB privilege. + +Most tests do not require special privileges. However, certain tests +may require Postgres SUPERUSER privilege. Such tests will normally be +skipped if the user lacks the privilege. + +Most tests do not require installation of supporting Postgres +packages. However, a few tests rely on the availability of Postgres +``contrib`` modules such as the `spi module +`_ or +procedural languages such as ``plperl``, ``plpython3u`` or ``plr``. + +On Windows, it is necessary to install Perl in order to run some of +the tests (most Linux or Unix variants already include it as part of +their normal distribution). The last time we checked, a suitable +choice appeared to be Strawberry Perl which can be downloaded from +http://strawberryperl.com/releases.html. However, the default +installation is placed in ``C:\strawberry`` and can hold a single Perl +version. Furthermore, some Postgres versions may be linked with +non-current Perl versions. It is recommended that the latest Perl +version be installed as this will usually give the fewest test +failures. See `this blog post +`_ +for more details. + +The COLLATION tests require the +``fr_FR.utf8`` locale (or ``French.France.1252`` language on Windows) +to be installed. + +Testing Checklist +----------------- + +The following is a summary list of steps needed to test Pyrseas on a +new machine. Refer to :ref:`development` for details on how to +accomplish a given installation task. "Package manager" refers to the +platform's package management system utility such as ``apt-get`` or +``yum``. Installation from PyPI can be done with ``pip``. Some +operations require administrative or superuser privileges, at either +the operating system or Postgres level. + + - Install Git using package manager or from + https://git-scm.com/download (on Windows, prefer Git Bash) + + - ``git clone git://github.com/perseas/Pyrseas.git`` + + - Install Python 3.7 or higher, using package manager or from + installers at https://www.python.org/downloads/. + + - Install Postgres 13, 12, 11 or 10, using package manager or + binary installers at https://www.postgresql.org/download/ + + .. note:: On Linux, make sure you install the contrib and plperl + packages, e.g., on Debian, postgresql-contrib-n and + postgresql-plperl-n (where `n` is the Postgres + version number) + + - Install Psycopg, using package manager, or from PyPI + (https://pypi.org/project/psycopg/). + + .. note:: On Windows, you may first want to install a version of + Microsoft Visual Studio from `here`_. An alternative + that may work is `MinGW `_. See + `these blog`_ `posts`_ for more details. + + .. _here: https://www.microsoft.com/en-us/download/developer-tools.aspx + + .. _these blog: https://pyrseas.wordpress.com/2012/09/25/testing-python-and-postgresql-on-windows-part-2/ + + .. _posts: https://pyrseas.wordpress.com/2012/09/28/testing-python-and-postgresql-on-windows-part-3/ + + - Install PyYAML, using package manager, or from PyPI + (https://pypi.org/project/PyYAML/) or + http://pyyaml.org/download/pyyaml/. + + - Install pytest, using package manager, or from PyPI + (https://pypi.org/project/pytest/). + + - Install Tox, using package manager, or from PyPI + (https://pypi.org/project/tox/) + + - On Windows, install Perl (see discussion above under + "Restrictions"). On Linux, usually Perl is already available. + + - As **postgres** user, using psql or pgAdmin, create a test user, + e.g., your name. The user running tests must have at a minimum + createdb privilege, in order to create the test database. To run + *all* the tests, the user also needs superuser privilege. + + - Create a Postgres password file, e.g., on Linux: ``~/.pgpass``, on + Windows: ``%APPDATA%\postgresql\pgpass.conf``. + + - Create directories to hold tablespaces, e.g., ``/extra/pg/13.0/ts1`` + on Linux, ``C:\\extra\\pg\\13.0\\ts1`` on Windows. The directories + need to be owned by the **postgres** user. This may be tricky on + older Windows versions, but the command ``cacls /E /G + postgres:F`` should suffice. Using ``psql``, create tablespaces + **ts1** and **ts2**, e.g., ``CREATE TABLESPACE ts1 LOCATION + ''`` (on Windows, you'll have to use, e.g., + ``E'C:\\dir\\ts1'``, to specify the directory). + + - Install the locale ``fr_FR.utf8`` on Linux/Unix or the language + ``French.France.1252`` on Windows. + + - On Debian and derivatives, this can be done with the command:: + + sudo dpkg-reconfigure locales + + - On Windows, open the Control Panel, select Date, Time, Language, + and Regional Options, then Regional and Language Options (or Add + other languages), click on the Advanced tab in the dialog and + then choose “French (France)” from the dropdown. Finally, click + OK and respond to any subsequent prompts to install the locale, + including rebooting the machine. + + - Change to the Pyrseas source directory (created by the second step above). + + - Define the ``PYTHONPATH`` environment variable to the Pyrseas source + directory, e.g., on Linux, ``export PYTHONPATH=$PWD``, on + Windows, ``set PYTHONPATH=%USERPROFILE%\somedir\Pyrseas``. + + - Define the ``PG1N0_PORT`` environment variables (where ``1N`` + represents the major Posgres version, e.g., 15, 11) to point to the + corresponding Postgres connection ports. + + - Invoke ``tox``. This will create virtualenvs in a ``.tox`` + subdirectory, + install Pyrseas and its prerequisites (Psycopg and PyYAML) into + each virtualenv and run the unit tests for each combination of + Postgres and Python. + +If you find any problems with the instructions above, please open an +issue on `GitHub `_. diff --git a/docs/textsearch.rst b/docs/textsearch.rst new file mode 100644 index 0000000..c03c45d --- /dev/null +++ b/docs/textsearch.rst @@ -0,0 +1,102 @@ +Text Search Objects +=================== + +.. module:: pyrseas.dbobject.textsearch + +The :mod:`textsearch` module defines eight classes: classes +:class:`TSConfiguration`, :class:`TSDictionary`, :class:`TSParser` and +:class:`TSTemplate` derived from :class:`DbSchemaObject`, and classes +:class:`TSConfigurationDict`, :class:`TSDictionaryDict`, +:class:`TSParserDict` and :class:`TSTemplateDict` derived from +:class:`DbObjectDict`. + +Text Search Configuration +------------------------- + +:class:`TSConfiguration` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search configuration +`_. + +.. autoclass:: TSConfiguration + +.. automethod:: TSConfiguration.to_map + +.. automethod:: TSConfiguration.create + +Text Search Configuration Dictionary +------------------------------------ + +:class:`TSConfigurationDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search configurations in a database. + +.. autoclass:: TSConfigurationDict + +.. automethod:: TSConfigurationDict.from_map + +Text Search Dictionary +---------------------- + +:class:`TSDictionary` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search dictionary +`_. + +.. autoclass:: TSDictionary + +.. automethod:: TSDictionary.create + +Text Search Dictionary Dictionary +--------------------------------- + +:class:`TSDictionaryDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a Python dictionary +that represents the collection of text search dictionaries in a +database. + +.. autoclass:: TSDictionaryDict + +.. automethod:: TSDictionaryDict.from_map + +Text Search Parser +------------------ + +:class:`TSParser` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search parser +`_. + +.. autoclass:: TSParser + +.. automethod:: TSParser.create + +Text Search Parser Dictionary +----------------------------- + +:class:`TSParserDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search parsers in a database. + +.. autoclass:: TSParserDict + +.. automethod:: TSParserDict.from_map + +Text Search Template +-------------------- + +:class:`TSTemplate` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search template +`_. + +.. autoclass:: TSTemplate + +.. automethod:: TSTemplate.create + +Text Search Template Dictionary +------------------------------- + +:class:`TSTemplateDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search templates in a database. + +.. autoclass:: TSTemplateDict + +.. automethod:: TSTemplateDict.from_map diff --git a/docs/trigger.rst b/docs/trigger.rst new file mode 100644 index 0000000..3d5e097 --- /dev/null +++ b/docs/trigger.rst @@ -0,0 +1,35 @@ +Triggers +======== + +.. module:: pyrseas.dbobject.trigger + +The :mod:`trigger` module defines two classes, :class:`Trigger` and +:class:`TriggerDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Trigger +------- + +:class:`Trigger` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a Postgres +`regular or constraint trigger +`_. + +.. autoclass:: Trigger + +.. automethod:: Trigger.identifier + +.. automethod:: Trigger.to_map + +.. automethod:: Trigger.create + +Trigger Dictionary +------------------ + +:class:`TriggerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of triggers in a database. + +.. autoclass:: TriggerDict + +.. automethod:: TriggerDict.from_map diff --git a/docs/type.rst b/docs/type.rst new file mode 100644 index 0000000..8482f9e --- /dev/null +++ b/docs/type.rst @@ -0,0 +1,120 @@ +Types and Domains +================= + +.. module:: pyrseas.dbobject.dbtype + +The :mod:`dbtype` module defines seven classes, :class:`DbType` +derived from :class:`DbSchemaObject`, :class:`BaseType`, +:class:`Composite`, :class:`Enum`, :class:`Domain` and :class:`Range` +derived from :class:`DbType`, and :class:`TypeDict` derived from and +:class:`DbObjectDict`. + +Database Type +------------- + +Class :class:`DbType` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a SQL type or +domain as defined in the Postgres `pg_type` catalog. + +.. autoclass:: DbType + +Base Type +--------- + +:class:`BaseType` is derived from :class:`~pyrseas.dbobject.DbType` +and represents a Postgres `user-defined base type +`_. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`TypeDict.from_map` has the following structure (not all fields +need be present):: + + {'type t1': + {'alignment': 'double', + 'analyze': 'analyze_func', + 'category': 'U', + 'delimiter': ',', + 'input': 'input_func', + 'internallength': 'variable', + 'output': 'output_func', + 'preferred': 'true', + 'receive': 'receive_func', + 'send': 'send_func', + 'storage': 'plain' + 'typmod_in': 'typmod_in_func', + 'typmod_out': 'typmod_out_func' + } + } + +.. autoclass:: BaseType + +.. automethod:: BaseType.to_map + +.. automethod:: BaseType.create + +.. automethod:: BaseType.drop + +Composite +--------- + +:class:`Composite` is derived from :class:`~pyrseas.dbobject.DbType` +and represents a standalone `composite type +`_. + +.. autoclass:: Composite + +.. automethod:: Composite.to_map + +.. automethod:: Composite.create + +.. automethod:: Composite.alter + +Enum +---- + +:class:`Enum` is derived from :class:`~pyrseas.dbobject.DbType` and +represents an `enumerated type +`_. + +.. autoclass:: Enum + +.. automethod:: Enum.create + +Domain +------ + +:class:`Domain` is derived from :class:`~pyrseas.dbobject.DbType` and +represents a `domain +`_. + +.. autoclass:: Domain + +.. automethod:: Domain.to_map + +.. automethod:: Domain.create + +Range +----- + +:class:`Range` is derived from :class:`~pyrseas.dbobject.DbType` and +represents a `Postgres range type +`_. + +.. autoclass:: Range + +.. automethod:: Range.to_map + +.. automethod:: Range.create + +.. automethod:: Range.alter + +Type Dictionary +--------------- + +:class:`TypeDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of domains and enums in a database. + +.. autoclass:: TypeDict + +.. automethod:: TypeDict.from_map diff --git a/docs/yamltodb.rst b/docs/yamltodb.rst new file mode 100644 index 0000000..fa242e4 --- /dev/null +++ b/docs/yamltodb.rst @@ -0,0 +1,129 @@ +yamltodb - YAML to Database +=========================== + +Name +---- + +yamltodb -- generate SQL statements to update a Postgres database to +match the schema specified in a YAML file + +Synopsis +-------- + +:: + + yamltodb [option...] dbname [spec] + +Description +----------- + +:program:`yamltodb` is a utility for generating SQL statements to +update a Postgres database so that it will match the schema +specified in an input `YAML `_ formatted +specification file. + +For example, given the input file shown under :doc:`dbtoyaml`, +:program:`yamltodb`, when run against a newly-created database, +outputs the following SQL statements:: + + CREATE SCHEMA s1; + ALTER SCHEMA s1 OWNER TO bob; + GRANT ALL ON SCHEMA s1 TO bob; + GRANT ALL ON SCHEMA s1 TO alice; + CREATE TABLE t1 ( + c1 integer NOT NULL, + c2 smallint, + c3 boolean DEFAULT false, + c4 text); + ALTER TABLE t1 OWNER TO alice; + CREATE TABLE s1.t2 ( + c21 integer NOT NULL, + c22 character varying(16)); + ALTER TABLE s1.t2 OWNER TO bob; + GRANT ALL ON TABLE s1.t2 TO bob; + GRANT SELECT ON TABLE s1.t2 TO PUBLIC; + GRANT INSERT, DELETE, UPDATE ON TABLE s1.t2 TO alice WITH GRANT OPTION; + GRANT INSERT ON TABLE s1.t2 TO carol; + ALTER TABLE t1 ADD CONSTRAINT t1_c2_check CHECK (c2 > 123); + ALTER TABLE t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1); + ALTER TABLE s1.t2 ADD CONSTRAINT t2_pkey PRIMARY KEY (c21); + ALTER TABLE t1 ADD CONSTRAINT t1_c2_fkey FOREIGN KEY (c2) REFERENCES s1.t2 (c21); + +Options +------- + +:program:`yamltodb` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +.. program:: yamltodb + +**dbname** + + Specifies the name of the database whose schema is to analyzed. + +**spec** + + Specifies the location of the YAML specification. If this is + omitted or specified as a single or double dash, the specification + is read from the program's standard input. However, if the + :option:`--multiple-files` option is used, that takes precedence. + +.. cmdoption:: -m, --multiple-files + + Specifies that input should be taken from YAML specification files + present in a two-level (metadata) directory tree. See `Multiple + File Output` under :doc:`dbtoyaml` for further details. + +.. cmdoption:: -n + --schema + + Compare only a schema matching `schema`. By default, all schemas + are compared. Multiple schemas can be compared by using multiple + :option:`-n` switches. + +.. cmdoption:: -1 + --single-transaction + + Wrap the generated statements in BEGIN/COMMIT. This ensures that + either all the statements complete successfully, or no changes are + applied. + +.. cmdoption:: -u, --update + + Execute the generated statements against the database mentioned in + **dbname**. This implies the :option:`--single-transaction` + option. + +.. cmdoption:: --revert + + Generate SQL in reversion mode, that is, to undo the changes that + would normally be generated. For example, if without this option, + the SQL would be a ``DROP TABLE``, the :option:`--revert` option + generates a ``CREATE TABLE`` with all the columns, constraints and + other objects associated with the table being dropped. + + This option is experimental and currently has only been + implemented for schemas and sequences. + +Examples +-------- + +Given a YAML file named ``moviesdb.yaml``, to generate SQL statements +to update a database called `mymovies`:: + + yamltodb mymovies moviesdb.yaml + +To generate the statements as above and immediately update `mymovies`:: + + yamltodb mymovies moviesdb.yaml | psql mymovies + +or:: + + yamltodb --update mymovies moviesdb.yaml + +To generate the statements directly from the output of +:program:`dbtoyaml` (against a different database), with statements +enclosed in a single transaction, and save the statements in a file +named ``mymovies.sql``:: + + dbtoyaml devmovies | yamltodb -1 mymovies -o mymovies.sql diff --git a/pyrseas/.vscode/launch.json b/pyrseas/.vscode/launch.json new file mode 100644 index 0000000..4adb9d5 --- /dev/null +++ b/pyrseas/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + {"args": [ + "-H", "192.168.128.7", "-U", "postgres", "test_db", "C:\\Projects\\new_dbsync\\db\\ddl\\test_db_schema.yaml" + ] + } + ] +} \ No newline at end of file diff --git a/pyrseas/__init__.py b/pyrseas/__init__.py new file mode 100644 index 0000000..9d1bb72 --- /dev/null +++ b/pyrseas/__init__.py @@ -0,0 +1 @@ +__version__ = '0.10.0' diff --git a/pyrseas/augment/__init__.py b/pyrseas/augment/__init__.py new file mode 100644 index 0000000..a2413c9 --- /dev/null +++ b/pyrseas/augment/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment + ~~~~~~~~~~~~~~~ + + This defines two low level classes. Most Database Augmenter + classes are derived from either DbAugment or DbAugmentDict. +""" + + +class DbAugment(object): + "A database augmentation object" + + keylist = ['name'] + """List of attributes that uniquely identify the object""" + + def __init__(self, **attrs): + """Initialize the augmentation object from a dictionary of attributes + + :param attrs: the dictionary of attributes + """ + for key, val in list(attrs.items()): + setattr(self, key, val) + + +class DbAugmentDict(dict): + """A dictionary of database augmentations, all of the same type""" + + cls = DbAugment + """The class, derived from :class:`DbAugment` that the objects belong to. + """ diff --git a/pyrseas/augment/audit.py b/pyrseas/augment/audit.py new file mode 100644 index 0000000..b1146e1 --- /dev/null +++ b/pyrseas/augment/audit.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.audit + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgAuditColumn derived from + DbAugment and CfgAuditColumnDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugment, DbAugmentDict +from pyrseas.dbobject import split_schema_obj + + +class CfgAuditColumn(DbAugment): + """An augmentation that adds automatically maintained audit columns""" + + keylist = ['name'] + + def apply(self, table, augdb): + """Apply audit columns to argument table. + + :param table: table to which columns/triggers will be added + :param augdb: augment dictionaries + """ + currdb = augdb.current + sch = table.schema + for col in self.columns: + augdb.columns[col].apply(table) + if hasattr(self, 'triggers'): + for trg in self.triggers: + augdb.triggers[trg].apply(table) + for newtrg in table.triggers: + fncsig = table.triggers[newtrg].procedure + (sch, fnc) = split_schema_obj(fncsig, table.schema) + if (sch, fncsig) not in currdb.functions: + newfunc = augdb.functions[fnc].apply( + sch, augdb.columns.col_trans_tbl, augdb) + # add new function to the current db + augdb.add_func(sch, newfunc) + augdb.add_lang(newfunc.language) + + +class CfgAuditColumnDict(DbAugmentDict): + "The collection of audit column augmentations" + + cls = CfgAuditColumn + + def __init__(self, config): + for aud in config: + self[aud] = CfgAuditColumn(name=aud, **config[aud]) + + def from_map(self, inaudcols): + """Initialize the dictionary of functions by converting the input map + + :param inaudcols: YAML map defining the audit column configuration + """ + for aud in inaudcols: + audcol = CfgAuditColumn(name=aud) + for attr in inaudcols[aud]: + if attr == 'columns': + audcol.columns = [col for col in inaudcols[aud][attr]] + elif attr == 'triggers': + audcol.triggers = [col for col in inaudcols[aud][attr]] + self[audcol.name] = audcol diff --git a/pyrseas/augment/column.py b/pyrseas/augment/column.py new file mode 100644 index 0000000..ee917c2 --- /dev/null +++ b/pyrseas/augment/column.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.column + ~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgColumn derived from + DbAugment and CfgColumnDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject.column import Column + + +class CfgColumn(DbAugment): + "A configuration column definition" + + keylist = ['name'] + + def apply(self, table): + """Add columns to the table passed in. + + :param table: table to which the columns will be added + """ + if self.name in table.column_names(): + for col in table.columns: + if col.name == self.name: + col.type = self.type + if hasattr(self, 'not_null'): + col.not_null = self.not_null + if hasattr(self, 'default'): + col.default = self.default + else: + dct = self.__dict__.copy() + dct.pop('name') + dct.pop('type') + newcol = Column(self.name, table.schema, table.name, 0, self.type, + **dct) + newcol._table = table + table.columns.append(newcol) + + +class CfgColumnDict(DbAugmentDict): + "The collection of configuration columns" + + cls = CfgColumn + + def __init__(self, config): + self.col_trans_tbl = [] + for col in config: + if not 'name' in config[col]: + config[col]['name'] = col + self[col] = CfgColumn(**config[col]) + self.col_trans_tbl.append(('{{%s}}' % col, self[col].name)) + + def from_map(self, incols): + """Initialize the dictionary of columns by converting the input dict + + :param incols: YAML dictionary defining the columns + """ + renames = False + for col in incols: + if col in self: + ccol = self[col] + else: + self[col] = ccol = CfgColumn(name=col) + for attr, val in list(incols[col].items()): + setattr(ccol, attr, val) + if attr == 'name': + renames = True + if renames: + self.col_trans_tbl = [('{{%s}}' % col, self[col].name) + for col in self] diff --git a/pyrseas/augment/function.py b/pyrseas/augment/function.py new file mode 100644 index 0000000..bbdb60f --- /dev/null +++ b/pyrseas/augment/function.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.function + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgFunction derived from + DbAugment and CfgFunctionDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject.function import Function + + +class CfgFunctionSource(DbAugment): + "A configuration function source or part thereof" + pass + + +class CfgFunctionTemplate(CfgFunctionSource): + "A configuration function source template" + + pass + + +class CfgFunctionSourceDict(DbAugmentDict): + + cls = CfgFunctionSource + + def __init__(self, cfg_templates): + for templ in cfg_templates: + src = cfg_templates[templ] + dct = {'source': src} + self[templ] = CfgFunctionTemplate(name=templ, **dct) + + def from_map(self, intempls): + """Initialize the dict of templates by converting the input list + + :param intempls: YAML list defining the function templates + """ + for templ in intempls: + self[templ] = CfgFunctionTemplate( + name=templ, source=intempls[templ]) + + +class CfgFunction(DbAugment): + "A configuration function definition" + + keylist = ['name', 'arguments'] + + def apply(self, schema, trans_tbl, augdb): + """Add a function to a given schema. + + :param schema: name of the schema in which to create the function + :param trans_tbl: translation table + :param augdb: augmenter dictionaries + """ + newdict = self.__dict__.copy() + newdict.pop('name') + newdict.pop('description') + newfunc = Function(self.name, schema, self.description, None, [], + **newdict) + src = newfunc.source + if '{{' in src and '}}' in src: + pref = src.find('{{') + prefix = src[:pref] + suf = src.find('}}') + suffix = src[suf + 2:] + tmplkey = src[pref + 2:suf] + if tmplkey not in augdb.funcsrcs: + if '{{'+tmplkey+'}}' not in [pat for (pat, repl) in trans_tbl]: + raise KeyError("Function template '%s' not found" % + tmplkey) + else: + newfunc.source = prefix + augdb.funcsrcs[tmplkey].source + \ + suffix + + for (pat, repl) in trans_tbl: + if '{{' in newfunc.source: + newfunc.source = newfunc.source.replace(pat, repl) + if '{{' in newfunc.name: + newfunc.name = newfunc.name.replace(pat, repl) + if '{{' in newfunc.description: + newfunc.description = newfunc.description.replace(pat, repl) + return newfunc + + +class CfgFunctionDict(DbAugmentDict): + "The collection of configuration functions" + + cls = CfgFunction + + def __init__(self, config): + for func in config: + fncdict = config[func] + paren = func.find('(') + (fnc, args) = (func[:paren], func[paren + 1:-1]) + fncname = fnc + dct = fncdict.copy() + if 'name' in dct: + fncname = dct['name'] + del dct['name'] + self[fnc] = CfgFunction(name=fncname, arguments=args, **dct) + + def from_map(self, infuncs): + """Initialize the dictionary of functions by converting the input list + + :param infuncs: YAML list defining the functions + """ + for func in infuncs: + paren = func.find('(') + (fnc, args) = (func[:paren], func[paren + 1:-1]) + if fnc in self: + cfnc = self[fnc] + else: + self[fnc] = cfnc = CfgFunction(name=fnc, arguments=args) + for attr, val in list(infuncs[func].items()): + setattr(cfnc, attr, val) diff --git a/pyrseas/augment/schema.py b/pyrseas/augment/schema.py new file mode 100644 index 0000000..a095b64 --- /dev/null +++ b/pyrseas/augment/schema.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.schema + ~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, AugSchema and AugSchemaDict, derived from + DbAugment and DbAugmentDict, respectively. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.augment.table import AugTable + + +class AugSchema(DbAugment): + """A database schema definition, i.e., a named collection of tables, + views, triggers and other schema objects.""" + + keylist = ['name'] + + def apply(self, augdb): + """Augment objects in a schema. + + :param augdb: the augmenter dictionaries + """ + for tbl in self.tables: + self.tables[tbl].apply(augdb) + + def add_func(self, func): + """Add a function to the schema if not already present + + :param func: the possibly new function + """ + sch = self.current + if not hasattr(sch, 'functions'): + sch.functions = {} + if func.name not in sch.functions: + sch.functions.update({func.name: func}) + + +class AugSchemaDict(DbAugmentDict): + "The collection of schemas in a database" + + cls = AugSchema + + def from_map(self, augmap, augdb): + """Initialize the dictionary of schemas by converting the augmenter map + + :param augmap: the input YAML map defining the augmentations + :param augdb: collection of dictionaries defining the augmentations + + Starts the recursive analysis of the input map and + construction of the internal collection of dictionaries + describing the database objects. + """ + for key in augmap: + (objtype, spc, sch) = key.partition(' ') + if spc != ' ' or objtype != 'schema': + raise KeyError("Unrecognized object type: %s" % key) + schema = self[sch] = AugSchema(name=sch) + inschema = augmap[key] + augtables = {} + augfuncs = {} + for key in inschema: + if key.startswith('table '): + augtables.update({key: inschema[key]}) + elif key.startswith('function '): + augfuncs.update({key: inschema[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + augdb.tables.from_map(schema, augtables, augdb) + + def link_current(self, schemas): + """Connect schemas to be augmented to actual database schemas + + :param schemas: schemas in current database + """ + for sch in self: + if not sch in schemas: + raise KeyError("Schema %s not in current database" % sch) + if not hasattr(self[sch], 'current'): + self[sch].current = schemas[sch] + + def link_refs(self, dbtables): + """Connect tables and functions to their respective schemas + + :param dbtables: dictionary of tables + + Fills in the `tables` dictionary for each schema by + traversing the `dbtables` dictionary. + """ + for (sch, tbl) in dbtables: + table = dbtables[(sch, tbl)] + assert self[sch] + schema = self[sch] + if isinstance(table, AugTable): + if not hasattr(schema, 'tables'): + schema.tables = {} + schema.tables.update({tbl: table}) diff --git a/pyrseas/augment/table.py b/pyrseas/augment/table.py new file mode 100644 index 0000000..335034f --- /dev/null +++ b/pyrseas/augment/table.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.table + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines three classes: AugDbClass derived from + DbAugment, AugTable derived from AugDbClass, and AugClassDict + derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment + + +class AugDbClass(DbAugment): + """A table, sequence or view""" + + keylist = ['schema', 'name'] + + +class AugTable(AugDbClass): + """A database table definition""" + + def apply(self, augdb): + """Augment tables in a schema. + + :param augdb: the augmenter dictionaries + """ + currtbl = augdb.current.tables[self.current.key()] + if hasattr(self, 'audit_columns'): + if self.audit_columns not in augdb.auditcols: + raise KeyError("Specification %s not in current configuration" + % self.audit_columns) + augdb.auditcols[self.audit_columns].apply(currtbl, augdb) + + +class AugClassDict(DbAugmentDict): + "The collection of tables and similar objects in a database" + + cls = AugDbClass + + def from_map(self, schema, inobjs, augdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param augdb: collection of dictionaries defining the augmentations + """ + for k in inobjs: + (objtype, spc, key) = k.partition(' ') + if spc != ' ' or objtype not in ['table']: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'table': + self[(schema.name, key)] = table = AugTable( + schema=schema.name, name=key) + intable = inobjs[k] + if not intable: + raise ValueError("Table '%s' has no specification" % k) + for attr in intable: + if attr == 'audit_columns': + setattr(table, attr, intable[attr]) + else: + raise KeyError("Unrecognized attribute '%s' for %s" + % (attr, k)) + else: + raise KeyError("Unrecognized object type: %s" % k) + + def link_current(self, tables): + """Connect tables to be augmented to actual database tables + + :param tables: tables in current schema + """ + for (sch, tbl) in self: + if not (sch, tbl) in tables: + raise KeyError("Table %s.%s not in current database" % ( + sch, tbl)) + if not hasattr(self[(sch, tbl)], 'current'): + self[(sch, tbl)].current = tables[(sch, tbl)] diff --git a/pyrseas/augment/trigger.py b/pyrseas/augment/trigger.py new file mode 100644 index 0000000..7bbab9b --- /dev/null +++ b/pyrseas/augment/trigger.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.trigger + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgTrigger derived from + DbAugment and CfgTriggerDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject import split_schema_obj +from pyrseas.dbobject.trigger import Trigger + + +class CfgTrigger(DbAugment): + "A configuration trigger definition" + + keylist = ['name'] + + def apply(self, table): + """Create a trigger for the table passed in. + + :param table: table on which the trigger will be created + """ + newtrg = Trigger(self.name, table.schema, table.name, + getattr(self, 'description', None), + self.procedure, self.timing, self.level, self.events) + newtrg._iscfg = True + if newtrg.name.startswith('{{table_name}}'): + newtrg.name = newtrg.name.replace(newtrg.name[:14], table.name) + newtrg._table = table + if not hasattr(table, 'triggers'): + table.triggers = {} + if hasattr(newtrg, 'procedure'): + if newtrg.procedure.startswith('{{table_name}}'): + newtrg.procedure = newtrg.procedure.replace( + newtrg.procedure[:14], table.name) + (sch, fnc) = split_schema_obj(newtrg.procedure) + if sch != table.schema: + newtrg.procedure = "%s.%s" % (table.schema, fnc) + table.triggers.update({newtrg.name: newtrg}) + + +class CfgTriggerDict(DbAugmentDict): + "The collection of configuration triggers" + + cls = CfgTrigger + + def __init__(self, config): + """Initialize internal configuration triggers""" + for trg in config: + self[trg] = CfgTrigger(**config[trg]) + + def from_map(self, intrigs): + """Initialize the dictionary of triggers by converting the input dict + + :param intrigs: YAML dictionary defining the triggers + """ + for trg in intrigs: + if trg in self: + ctrg = self[trg] + else: + self[trg] = ctrg = CfgTrigger(name=trg) + for attr, val in list(intrigs[trg].items()): + setattr(ctrg, attr, val) diff --git a/pyrseas/augmentdb.py b/pyrseas/augmentdb.py new file mode 100644 index 0000000..30ad9d9 --- /dev/null +++ b/pyrseas/augmentdb.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augmentdb + ~~~~~~~~~~~~~~~~~ + + An `AugmentDatabase` is initialized with a DbConnection object. + It consists of two "dictionary" container objects, each holding + various dictionary objects. The `db` Dicts object (inherited from + its parent class), defines the database schemas, including their + tables and other objects, by querying the system catalogs. The + `adb` AugDicts object defines the augmentation schemas and the + configuration objects based on the aug_map supplied to the `apply` + method. +""" +from pyrseas.database import Database +from pyrseas.dbobject.language import Language +from pyrseas.augment.schema import AugSchemaDict +from pyrseas.augment.table import AugClassDict +from pyrseas.augment.column import CfgColumnDict +from pyrseas.augment.function import CfgFunctionDict, CfgFunctionSourceDict +from pyrseas.augment.trigger import CfgTriggerDict +from pyrseas.augment.audit import CfgAuditColumnDict + + +def cfg_section(config, section): + "Return the configuration section if present, else an empty dict" + return config[section] if section in config else {} + + +class AugmentDatabase(Database): + """A database that is to be augmented""" + + class AugDicts(object): + """A holder for dictionaries (maps) describing augmentations""" + + def __init__(self, config): + """Initialize the various DbAugmentDict-derived dictionaries + + :param config: configuration dictionary + """ + self.schemas = AugSchemaDict() + self.tables = AugClassDict() + self.columns = CfgColumnDict(cfg_section(config, 'columns')) + self.funcsrcs = CfgFunctionSourceDict( + cfg_section(config, 'function_templates')) + self.functions = CfgFunctionDict(cfg_section(config, 'functions')) + self.triggers = CfgTriggerDict(cfg_section(config, 'triggers')) + self.auditcols = CfgAuditColumnDict( + cfg_section(config, 'audit_columns')) + + def _link_refs(self): + """Link related objects""" + self.schemas.link_refs(self.tables) + + def _link_current(self, db): + """Link augment objects to current catalog objects""" + self.current = db + self.schemas.link_current(db.schemas) + self.tables.link_current(db.tables) + + def add_func(self, schema, function): + """Add a function to a schema if not already present + + :param schema: schema name + :param function: the possibly new function + """ + if schema in self.schemas: + self.schemas[schema].add_func(function) + elif schema in self.current.schemas: + sch = self.current.schemas[schema] + if not hasattr(sch, 'functions'): + sch.functions = {} + if function.name not in sch.functions: + sch.functions.update({function.name: function}) + + def add_lang(self, lang): + """Add a language if not already present + + :param lang: the possibly new language + """ + if lang not in self.current.languages: + self.current.languages[lang] = Language(lang) + + def from_augmap(self, aug_map): + """Populate the augment objects from the input augment map + + :param aug_map: a YAML map defining the desired augmentations + + The `adb` holder is populated by various DbAugmentDict-derived + classes by traversing the YAML augmentation map. The objects + in the dictionary are then linked to related objects, e.g., + tables are linked to the schemas they belong. + """ + self.adb = self.AugDicts(cfg_section(self.config, 'augmenter')) + aug_schemas = {} + for key in aug_map: + if key == 'augmenter': + self._from_cfgmap(aug_map[key]) + elif key.startswith('schema '): + aug_schemas.update({key: aug_map[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + self.adb.schemas.from_map(aug_schemas, self.adb) + self.adb._link_refs() + self.adb._link_current(self.db) + + def _from_cfgmap(self, cfg_map): + """Populate configuration objects from the input configuration map + + :param cfg_map: a YAML map defining augmentation configuration + + The augmentations dictionary is populated by various + DbAugmentDict-derived classes by traversing the YAML + configuration map. + """ + for key in cfg_map: + if key == 'columns': + self.adb.columns.from_map(cfg_map[key]) + elif key in ['function_templates', 'function_segments']: + self.adb.funcsrcs.from_map(cfg_map[key]) + elif key == 'functions': + self.adb.functions.from_map(cfg_map[key]) + elif key == 'triggers': + self.adb.triggers.from_map(cfg_map[key]) + elif key == 'audit_columns': + self.adb.auditcols.from_map(cfg_map[key]) + else: + raise KeyError("Expected typed object, found '%s'" % key) + + def apply(self, aug_map): + """Apply augmentations to an existing database + + :param aug_map: a YAML map defining the desired augmentations + + Merges an existing database definition, as fetched from the + catalogs, with an input YAML defining augmentations on various + objects and an optional configuration map or the predefined + configuration. + """ + if not self.db: + self.from_catalog() + self.from_augmap(aug_map) + for sch in self.adb.schemas: + self.adb.schemas[sch].apply(self.adb) + return self.to_map() diff --git a/pyrseas/cmdargs.py b/pyrseas/cmdargs.py new file mode 100644 index 0000000..ef23796 --- /dev/null +++ b/pyrseas/cmdargs.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Utility module for command line argument parsing""" + +import os +from argparse import ArgumentParser, FileType +import getpass + +import yaml + +from pyrseas.config import Config + +_cfg = None + +HELP_TEXT = { + 'host': "database server host or socket directory", + 'port': "database server port number", + 'username': "database user name" +} + + +def _help_dflt(arg, config): + kwdargs = {'help': HELP_TEXT[arg]} + if arg in config: + kwdargs['help'] += " (default %(default)s)" + kwdargs['default'] = config[arg] + return kwdargs + + +def _repo_path(cfg, key=None): + """Return path to root directory of repository or subdirectory + + :return: path + """ + repo = cfg['repository'] + if 'path' in repo: + path = repo['path'] + else: + path = os.getcwd() + subdir = '' if key is None else repo[key] + return os.path.normpath(os.path.join(path, subdir)) + + +def cmd_parser(description, version): + """Create command line argument parser with common PostgreSQL options + + :param description: text to display before the argument help + :param version: version of the caller + :return: the created parser + """ + global _cfg + + parent = ArgumentParser(add_help=False) + parent.add_argument('dbname', help='database name') + group = parent.add_argument_group('Connection options') + if _cfg is None: + _cfg = Config() + dbcfg = _cfg['database'] if 'database' in _cfg else {} + group.add_argument('-H', '--host', **_help_dflt('host', dbcfg)) + group.add_argument('-p', '--port', type=int, **_help_dflt('port', dbcfg)) + group.add_argument('-U', '--username', **_help_dflt('username', dbcfg)) + group.add_argument('-W', '--password', action="store_true", + help="force password prompt") + parent.add_argument('-c', '--config', type=FileType('r'), + help="configuration file path") + parent.add_argument('-r', '--repository', default=_repo_path(_cfg), + help="root of repository (default %(default)s)") + parent.add_argument('-o', '--output', type=FileType('w'), + help="output file name (default stdout)") + parser = ArgumentParser(parents=[parent], description=description) + parser.add_argument('--version', action='version', + version='%(prog)s ' + '%s' % version) + return parser + + +def parse_args(parser): + """Parse command line arguments and return configuration object + + :param parser: ArgumentParser created by cmd_parser + :return: a Configuration object + """ + arg_opts = parser.parse_args() + args = vars(arg_opts) + for key in ['database', 'files']: + if key not in _cfg: + _cfg[key] = {} + + def tfr(prim, key, val): + _cfg[prim][key] = val + del args[key] + + for key in ['dbname', 'host', 'port', 'username']: + tfr('database', key, args[key]) + tfr('database', 'password', + (getpass.getpass() if args['password'] else None)) + + for key in ['output', 'config']: + tfr('files', key, args[key]) + + if 'config' in _cfg['files'] and _cfg['files']['config']: + _cfg.merge(yaml.safe_load(_cfg['files']['config'])) + if 'repository' in args: + if args['repository'] != os.getcwd(): + _cfg['repository']['path'] = args['repository'] + del args['repository'] + + _cfg['files']['metadata_path'] = _repo_path(_cfg, 'metadata') + _cfg['files']['data_path'] = _repo_path(_cfg, 'data') + + _cfg['options'] = arg_opts + return _cfg diff --git a/pyrseas/config.py b/pyrseas/config.py new file mode 100644 index 0000000..93ab43a --- /dev/null +++ b/pyrseas/config.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""Utility module for configuration file parsing""" + +import os +import sys + +import yaml + + +CFG_FILE = os.environ.get("PYRSEAS_CONFIG_FILE", "config.yaml") + + +def _home_dir(): + if sys.platform == 'win32': + dir = os.getenv('APPDATA', '') + else: + dir = os.path.join(os.environ['HOME'], '.config') + return os.path.abspath(dir) + + +def _load_cfg(cfgdir): + cfgpath = '' + cfg = {} + if cfgdir is not None: + if os.path.isdir(cfgdir): + cfgpath = os.path.join(cfgdir, CFG_FILE) + elif os.path.isfile(cfgdir): + cfgpath = cfgdir + if os.path.exists(cfgpath): + with open(cfgpath) as f: + cfg = yaml.safe_load(f) + return cfg + + +class Config(dict): + "A configuration dictionary" + + def __init__(self, sys_only=False): + self.update(_load_cfg( + os.environ.get("PYRSEAS_SYS_CONFIG", os.path.abspath(os.path.join( + os.path.dirname(__file__)))))) + if sys_only: + return + self.merge(_load_cfg(os.environ.get("PYRSEAS_USER_CONFIG", + os.path.join(_home_dir(), 'pyrseas')))) + if 'repository' in self and 'path' in self['repository']: + cfgpath = self['repository']['path'] + else: + cfgpath = os.getcwd() + self.merge(_load_cfg(cfgpath)) + + def merge(self, cfg): + """Merge extra configuration + + :param cfg: extra configuration (dict) + """ + for key, val in list(cfg.items()): + if key in self: + self[key].update(val) + else: + self[key] = val diff --git a/pyrseas/config.yaml b/pyrseas/config.yaml new file mode 100644 index 0000000..018af66 --- /dev/null +++ b/pyrseas/config.yaml @@ -0,0 +1,160 @@ +augmenter: + audit_columns: + created_date_only: + columns: + - created_date + default: + columns: + - modified_by_user + - modified_timestamp + triggers: + - audit_default + modified_only: + columns: + - modified_timestamp + triggers: + - audit_modified_only + full: + columns: + - created_by_user + - created_by_ip_address + - created_timestamp + - modified_by_user + - modified_by_ip_address + - modified_timestamp + triggers: + - audit_full + columns: + created_by_ip_address: + not_null: true + type: inet + created_by_user: + not_null: true + type: character varying(63) + created_date: + default: ('now'::text)::date + not_null: true + type: date + created_timestamp: + not_null: true + type: timestamp with time zone + modified_by_ip_address: + not_null: true + type: inet + modified_by_user: + not_null: true + type: character varying(63) + modified_timestamp: + not_null: true + type: timestamp with time zone + function_templates: + functempl_audit_default: |2- + + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + RETURN NEW; + END + functempl_audit_full: |2- + + DECLARE + user_name name := pyrseas.get_session_variable('audit_user', + SESSION_USER::varchar); + ip_address inet := pyrseas.get_session_variable('audit_ip_address', + COALESCE(pg_catalog.inet_client_addr()::varchar, '0.0.0.0')); + BEGIN + IF TG_OP = 'INSERT' THEN + NEW.{{created_timestamp}} = CURRENT_TIMESTAMP; + NEW.{{created_by_user}} = user_name; + NEW.{{created_by_ip_address}} = ip_address; + ELSIF TG_OP = 'UPDATE' THEN + NEW.{{created_timestamp}} = OLD.created_timestamp; + NEW.{{created_by_user}} = OLD.created_by_user; + NEW.{{created_by_ip_address}} = OLD.created_by_ip_address; + END IF; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + NEW.{{modified_by_user}} = user_name; + NEW.{{modified_by_ip_address}} = ip_address; + RETURN NEW; + END + functempl_audit_modified: |2- + + BEGIN + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + RETURN NEW; + END + functions: + audit_default(): + description: |- + Provides modified_by_user and modified_timestamp values for audit + columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_default}}' + audit_full(): + description: |- + Provides created/modified values for audit columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_full}}' + audit_modified(): + description: |- + Provides modified_timestamp values for audit columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_modified}}' + schema pyrseas: + function get_session_variable(var_name character varying): + language: plpgsql + returns: character varying + source: |2 + BEGIN + RETURN pg_catalog.current_setting('pyrseas.' || var_name); + END; + function get_session_variable(var_name character varying, default_value character varying): + language: plpgsql + returns: character varying + source: |2 + BEGIN + RETURN pg_catalog.current_setting('pyrseas.' || var_name); + EXCEPTION + WHEN undefined_object THEN + RETURN default_value; + END; + function set_session_variable(var_name character varying, var_value character varying): + language: plpgsql + returns: void + source: |2 + BEGIN + PERFORM pg_catalog.set_config('pyrseas.' || var_name, var_value, false); + END; + triggers: + audit_default: + events: + - update + level: row + name: '{{table_name}}_20_audit_default' + procedure: audit_default() + timing: before + audit_full: + events: + - insert + - update + level: row + name: '{{table_name}}_20_audit_full' + procedure: audit_full() + timing: before + audit_modified_only: + events: + - insert + - update + level: row + name: '{{table_name}}_20_audit_modified_only' + procedure: audit_modified() + timing: before +repository: + metadata: metadata + data: metadata diff --git a/pyrseas/database.py b/pyrseas/database.py new file mode 100644 index 0000000..104335f --- /dev/null +++ b/pyrseas/database.py @@ -0,0 +1,650 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.database + ~~~~~~~~~~~~~~~~ + + A `Database` is initialized with a DbConnection object. It + consists of one or two `Dicts` objects, each holding various + dictionary objects. The `db` Dicts object defines the database + schemas, including their tables and other objects, by querying the + system catalogs. The `ndb` Dicts object defines the schemas based + on the `input_map` supplied to the `from_map` method. +""" +import os +import sys +from operator import itemgetter +from collections import defaultdict, deque +import yaml + +from pyrseas.lib.dbconn import DbConnection + +from pyrseas.yamlutil import yamldump +from pyrseas.dbobject import fetch_reserved_words, DbObjectDict, DbSchemaObject +from pyrseas.dbobject.language import LanguageDict +from pyrseas.dbobject.cast import CastDict +from pyrseas.dbobject.schema import SchemaDict +from pyrseas.dbobject.dbtype import TypeDict +from pyrseas.dbobject.table import ClassDict +from pyrseas.dbobject.column import ColumnDict +from pyrseas.dbobject.constraint import ConstraintDict +from pyrseas.dbobject.index import IndexDict +from pyrseas.dbobject.function import ProcDict +from pyrseas.dbobject.operator import OperatorDict +from pyrseas.dbobject.operclass import OperatorClassDict +from pyrseas.dbobject.operfamily import OperatorFamilyDict +from pyrseas.dbobject.rule import RuleDict +from pyrseas.dbobject.trigger import TriggerDict +from pyrseas.dbobject.conversion import ConversionDict +from pyrseas.dbobject.textsearch import TSConfigurationDict, TSDictionaryDict +from pyrseas.dbobject.textsearch import TSParserDict, TSTemplateDict +from pyrseas.dbobject.foreign import ForeignDataWrapperDict +from pyrseas.dbobject.foreign import ForeignServerDict, UserMappingDict +from pyrseas.dbobject.foreign import ForeignTableDict +from pyrseas.dbobject.extension import ExtensionDict +from pyrseas.dbobject.collation import CollationDict +from pyrseas.dbobject.eventtrig import EventTriggerDict + + +def flatten(lst): + "Flatten a list possibly containing lists to a single list" + for elem in lst: + if isinstance(elem, list) and not isinstance(elem, str): + for subelem in flatten(elem): + yield subelem + else: + yield elem + + +class CatDbConnection(DbConnection): + """A database connection, specialized for querying catalogs""" + + def connect(self): + """Connect to the database""" + super(CatDbConnection, self).connect() + schs = self.fetchall("SELECT current_schemas(false)") + addschs = [sch for sch in schs[0]["current_schemas"] if sch != "public"] + srch_path = "pg_catalog" + if addschs: + srch_path += ", " + ", ".join(addschs) + self.execute("set search_path to %s" % srch_path) + self.commit() + self._version = self.conn.info.server_version + + @property + def version(self): + "The server's version number" + if self.conn is None: + self.connect() + return self._version + + +class Database(object): + """A database definition, from its catalogs and/or a YAML spec.""" + + class Dicts(object): + """A holder for dictionaries (maps) describing a database""" + + def __init__(self, dbconn=None, single_db=False): + """Initialize the various DbObjectDict-derived dictionaries + + :param dbconn: a DbConnection object + """ + self.schemas = SchemaDict(dbconn) + self.extensions = ExtensionDict(dbconn) + self.languages = LanguageDict(dbconn) + self.casts = CastDict(dbconn) + self.types = TypeDict(dbconn) + self.tables = ClassDict(dbconn) + self.columns = ColumnDict(dbconn) + self.constraints = ConstraintDict(dbconn) + self.indexes = IndexDict(dbconn) + self.functions = ProcDict(dbconn) + self.operators = OperatorDict(dbconn) + self.operclasses = OperatorClassDict(dbconn) + self.operfams = OperatorFamilyDict(dbconn) + self.rules = RuleDict(dbconn) + self.triggers = TriggerDict(dbconn) + self.conversions = ConversionDict(dbconn) + self.tstempls = TSTemplateDict(dbconn) + self.tsdicts = TSDictionaryDict(dbconn) + self.tsparsers = TSParserDict(dbconn) + self.tsconfigs = TSConfigurationDict(dbconn) + self.fdwrappers = ForeignDataWrapperDict(dbconn) + self.servers = ForeignServerDict(dbconn) + self.usermaps = UserMappingDict(dbconn) + self.ftables = ForeignTableDict(dbconn) + self.collations = CollationDict(dbconn) + self.eventtrigs = EventTriggerDict(dbconn) + + # Populate a map from system catalog to the respective dict + self._catalog_map = {} + for _, d in self.all_dicts(single_db): + if d.cls.catalog is not None: + self._catalog_map[d.cls.catalog] = d + + # Map from objects extkey to their (dict name, key) + self._extkey_map = {} + + def _get_by_extkey(self, extkey): + """Return any database item from its extkey + + Note: probably doesn't work for all the objects, e.g. constraints + may clash because two in different tables have different extkeys. + However this shouldn't matter as such objects are generated as part + of the containing one and they should be returned by the + `get_implied_deps()` implementation of specific classes (which + would look for the object in by key in the right dict instead, + (e.g. check `Domain.get_implied_deps()` implementation. + + """ + try: + return self._extkey_map[extkey] + except KeyError: + # TODO: Likely it's the first time we call this function so + # let's warm up the cache. But we should really define the life + # cycle of this object as trying and catching KeyError on it is + # *very* expensive! + for _, d in self.all_dicts(): + for obj in list(d.values()): + self._extkey_map[obj.extern_key()] = obj + + return self._extkey_map[extkey] + + def all_dicts(self, non_empty=False): + """Iterate over the DbObjectDict-derived dictionaries returning + an ordered list of tuples (dict name, DbObjectDict object). + + :param non_empty: do not include empty dicts + + :return: list of tuples + """ + rv = [] + for attr in self.__dict__: + d = getattr(self, attr) + if non_empty and len(d) == 0: + continue + if isinstance(d, DbObjectDict): + # skip ColumnDict as not needed for dependency tracking + # and internally has lists, not objects + if not isinstance(d, ColumnDict): + rv.append((attr, d)) + + # first return the dicts for non-schema objects, then the + # others, each group sorted alphabetically. + rv.sort(key=lambda pair: (issubclass(pair[1].cls, DbSchemaObject), + pair[1].cls.__name__)) + + return rv + + def dbobjdict_from_catalog(self, catalog): + """Given a catalog name, return corresponding DbObjectDict + + :param catalog: full name of a pg_ catalog + :return: DbObjectDict object + """ + return self._catalog_map.get(catalog) + + def find_type(self, name): + """Return a db type given a qualname + + Note that tables and views are types too. + """ + rv = self.types.find(name) + if rv is not None: + return rv + + rv = self.tables.find(name) + return rv + + def __init__(self, config): + """Initialize the database + + :param config: configuration dictionary + """ + db = config['database'] + self.dbconn = CatDbConnection(db['dbname'], db['username'], + db['password'], db['host'], db['port']) + self.db = None + self.config = config + + def _link_refs(self, db): + """Link related objects""" + langs = [] + if self.dbconn.version >= 90100: + langs = [lang["lanname"] for lang in self.dbconn.fetchall( + """SELECT lanname FROM pg_language l + JOIN pg_depend p ON (l.oid = p.objid) + WHERE deptype = 'e' """)] + db.languages.link_refs(db.functions, langs) + copycfg = {} + if 'datacopy' in self.config: + copycfg = self.config['datacopy'] + db.schemas.link_refs(db, copycfg) + db.tables.link_refs(db.columns, db.constraints, db.indexes, db.rules, + db.triggers) + db.functions.link_refs(db.types) + db.fdwrappers.link_refs(db.servers) + db.servers.link_refs(db.usermaps) + db.ftables.link_refs(db.columns) + db.types.link_refs(db.columns, db.constraints, db.functions) + db.constraints.link_refs(db) + + def _build_dependency_graph(self, db, dbconn): + """Build the dependency graph of the database objects + + :param db: dictionary of dictionary of all objects + :param dbconn: a DbConnection object + """ + alldeps = defaultdict(list) + + # This query wanted to be simple. it got complicated because + # we don't handle indexes together with the other pg_class + # but in their own pg_index place (so fetch i1, i2) + # "Normal" dependencies, but excluding system objects + # (objid < 16384 and refobjid < 16384) + query = """SELECT DISTINCT + CASE WHEN i1.indexrelid IS NOT NULL + THEN 'pg_index'::regclass + ELSE classid::regclass END AS class_name, objid, + CASE WHEN i2.indexrelid IS NOT NULL + THEN 'pg_index'::regclass + ELSE refclassid::regclass END AS refclass, refobjid + FROM pg_depend + LEFT JOIN pg_index i1 ON classid = 'pg_class'::regclass + AND objid = i1.indexrelid + LEFT JOIN pg_index i2 + ON refclassid = 'pg_class'::regclass + AND refobjid = i2.indexrelid + WHERE deptype = 'n' + AND NOT (objid < 16384 AND refobjid < 16384)""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['objid']].append( + (r['refclass'], r['refobjid'])) + + # The dependencies across views is not in pg_depend. We have to + # parse the rewrite rule. "ev_class >= 16384" is to exclude + # system views. + query = r"""SELECT DISTINCT 'pg_class' AS class_name, ev_class, + CASE WHEN depid[1] = 'relid' THEN 'pg_class' + WHEN depid[1] = 'funcid' THEN 'pg_proc' + END AS refclass, depid[2]::oid AS refobjid + FROM (SELECT ev_class, regexp_matches(ev_action, + ':(relid|funcid)\s+(\d+)', 'g') AS depid + FROM pg_rewrite + WHERE rulename = '_RETURN' + AND ev_class >= 16384) x + LEFT JOIN pg_class c + ON (depid[1], depid[2]::oid) = ('relid', c.oid) + LEFT JOIN pg_namespace cs ON cs.oid = relnamespace + LEFT JOIN pg_proc p + ON (depid[1], depid[2]::oid) = ('funcid', p.oid) + LEFT JOIN pg_namespace ps ON ps.oid = pronamespace + WHERE ev_class <> depid[2]::oid + AND coalesce(cs.nspname, ps.nspname) + NOT IN ('information_schema', 'pg_catalog')""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['ev_class']].append( + (r['refclass'], r['refobjid'])) + + # Add the dependencies between a table and other objects through the + # columns defaults + query = """SELECT 'pg_class' AS class_name, adrelid, + d.refclassid::regclass, d.refobjid + FROM pg_attrdef ad JOIN pg_depend d + ON classid = 'pg_attrdef'::regclass AND objid = ad.oid + AND deptype = 'n'""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['adrelid']].append( + (r['refclassid'], r['refobjid'])) + + for (stbl, soid), deps in list(alldeps.items()): + sdict = db.dbobjdict_from_catalog(stbl) + if sdict is None or len(sdict) == 0: + continue + src = sdict.by_oid.get(soid) + if src is None: + continue + for ttbl, toid in deps: + tdict = db.dbobjdict_from_catalog(ttbl) + if tdict is None or len(tdict) == 0: + continue + tgt = tdict.by_oid.get(toid) + if tgt is None: + continue + src.depends_on.append(tgt) + + def _trim_objects(self, schemas): + """Remove unwanted schema objects + + :param schemas: list of schemas to keep + """ + for objtype in ['types', 'tables', 'constraints', 'indexes', + 'functions', 'operators', 'operclasses', 'operfams', + 'rules', 'triggers', 'conversions', 'tstempls', + 'tsdicts', 'tsparsers', 'tsconfigs', 'extensions', + 'collations', 'eventtrigs']: + objdict = getattr(self.db, objtype) + for obj in list(objdict.keys()): + # obj[0] is the schema name in all these dicts + if obj[0] not in schemas: + del objdict[obj] + for sch in list(self.db.schemas.keys()): + if sch not in schemas: + del self.db.schemas[sch] + # exclude database-wide objects + self.db.languages = LanguageDict() + self.db.casts = CastDict() + + def from_catalog(self, single_db=False): + """Populate the database objects by querying the catalogs + + :param single_db: populating only this database? + + The `db` holder is populated by various DbObjectDict-derived + classes by querying the catalogs. A dependency graph is + constructed by querying the pg_depend catalog. The objects in + the dictionary are then linked to related objects, e.g., + columns are linked to the tables they belong. + """ + self.db = self.Dicts(self.dbconn, single_db) + self._build_dependency_graph(self.db, self.dbconn) + if self.dbconn.conn: + self.dbconn.conn.close() + self._link_refs(self.db) + + def from_map(self, input_map, langs=None): + """Populate the new database objects from the input map + + :param input_map: a YAML map defining the new database + :param langs: list of language templates + + The `ndb` holder is populated by various DbObjectDict-derived + classes by traversing the YAML input map. The objects in the + dictionary are then linked to related objects, e.g., columns + are linked to the tables they belong. + """ + self.ndb = self.Dicts() + input_schemas = {} + input_extens = {} + input_langs = {} + input_casts = {} + input_fdws = {} + input_ums = {} + input_evttrigs = {} + for key in input_map: + if key.startswith('schema '): + input_schemas.update({key: input_map[key]}) + elif key.startswith('extension '): + input_extens.update({key: input_map[key]}) + elif key.startswith('language '): + input_langs.update({key: input_map[key]}) + elif key.startswith('cast '): + input_casts.update({key: input_map[key]}) + elif key.startswith('foreign data wrapper '): + input_fdws.update({key: input_map[key]}) + elif key.startswith('user mapping for '): + input_ums.update({key: input_map[key]}) + elif key.startswith('event trigger '): + input_evttrigs.update({key: input_map[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + self.ndb.extensions.from_map(input_extens, self.ndb) + self.ndb.languages.from_map(input_langs) + self.ndb.schemas.from_map(input_schemas, self.ndb) + self.ndb.casts.from_map(input_casts, self.ndb) + self.ndb.fdwrappers.from_map(input_fdws, self.ndb) + self.ndb.eventtrigs.from_map(input_evttrigs, self.ndb) + self._link_refs(self.ndb) + + def map_from_dir(self): + """Read the database maps starting from the metadata directory + + :return: dictionary + """ + metadata_dir = self.config['files']['metadata_path'] + if not os.path.isdir(metadata_dir): + sys.exit("Metadata directory '%s' doesn't exist" % metadata_dir) + + def load(subdir, obj): + with open(os.path.join(subdir, obj), 'r') as f: + objmap = yaml.safe_load(f) + return objmap if isinstance(objmap, dict) else {} + + inmap = {} + for entry in os.listdir(metadata_dir): + if entry.endswith('.yaml'): + if entry.startswith('database.'): + continue + if not entry.startswith('schema.'): + inmap.update(load(metadata_dir, entry)) + else: + # skip over unknown files/dirs + if not entry.startswith('schema.'): + continue + # read schema.xxx.yaml first + schmap = load(metadata_dir, entry + '.yaml') + assert(len(schmap) == 1) + key = list(schmap.keys())[0] + inmap.update({key: {}}) + subdir = os.path.join(metadata_dir, entry) + if os.path.isdir(subdir): + for schobj in os.listdir(subdir): + schmap[key].update(load(subdir, schobj)) + inmap.update(schmap) + + return inmap + + def to_map(self, quote_reserved=True): + """Convert the db maps to a single hierarchy suitable for YAML + + :param quote_reserved: fetch reserved words + :return: a YAML-suitable dictionary (without any Python objects) + """ + if not self.db: + self.from_catalog(True) + + opts = self.config['options'] + + def mkdir_parents(dir): + head, tail = os.path.split(dir) + if head and not os.path.isdir(head): + mkdir_parents(head) + if tail: + os.mkdir(dir) + + if opts.multiple_files: + opts.metadata_dir = self.config['files']['metadata_path'] + if not os.path.exists(opts.metadata_dir): + mkdir_parents(opts.metadata_dir) + dbfilepath = os.path.join(opts.metadata_dir, 'database.%s.yaml' % + self.dbconn.dbname) + if os.path.exists(dbfilepath): + with open(dbfilepath, 'r') as f: + objmap = yaml.safe_load(f) + for obj, val in list(objmap.items()): + if isinstance(val, dict): + dirpath = '' + for schobj, fpath in list(val.items()): + filepath = os.path.join(opts.metadata_dir, fpath) + if os.path.exists(filepath): + os.remove(filepath) + if schobj == 'schema': + (dirpath, ext) = os.path.splitext(filepath) + if os.path.exists(dirpath): + os.rmdir(dirpath) + else: + filepath = os.path.join(opts.metadata_dir, val) + if (os.path.exists(filepath)): + os.remove(filepath) + if quote_reserved: + fetch_reserved_words(self.dbconn) + + dbmap = self.db.extensions.to_map(self.db, opts) + dbmap.update(self.db.languages.to_map(self.db, opts)) + dbmap.update(self.db.casts.to_map(self.db, opts)) + dbmap.update(self.db.fdwrappers.to_map(self.db, opts)) + dbmap.update(self.db.eventtrigs.to_map(self.db, opts)) + if 'datacopy' in self.config: + opts.data_dir = self.config['files']['data_path'] + if not os.path.exists(opts.data_dir): + mkdir_parents(opts.data_dir) + dbmap.update(self.db.schemas.to_map(self.db, opts)) + + if opts.multiple_files: + with open(dbfilepath, 'w') as f: + f.write(yamldump(dbmap)) + + return dbmap + + def diff_map(self, input_map, quote_reserved=True): + """Generate SQL to transform an existing database + + :param input_map: a YAML map defining the new database + :param quote_reserved: fetch reserved words + :return: list of SQL statements + + Compares the existing database definition, as fetched from the + catalogs, to the input YAML map and generates SQL statements + to transform the database into the one represented by the + input. + """ + from .dbobject.table import Table + + if not self.db: + self.from_catalog() + opts = self.config['options'] + if opts.schemas: + schlist = ['schema ' + sch for sch in opts.schemas] + for sch in list(input_map.keys()): + if sch not in schlist and sch.startswith('schema '): + del input_map[sch] + self._trim_objects(opts.schemas) + + # quote_reserved is only set to False by most tests + if quote_reserved: + fetch_reserved_words(self.dbconn) + + self.from_map(input_map) + if opts.revert: + (self.db, self.ndb) = (self.ndb, self.db) + del self.ndb.schemas['pg_catalog'] + self.db.languages.dbconn = self.dbconn + + # First sort the objects in the new db in dependency order + new_objs = [] + for _, d in self.ndb.all_dicts(): + pairs = list(d.items()) + pairs.sort() + new_objs.extend(list(map(itemgetter(1), pairs))) + + new_objs = self.dep_sorted(new_objs, self.ndb) + + # Then generate the sql for all the objects, walking in dependency + # order over all the db objects + + stmts = [] + for new in new_objs: + d = self.db.dbobjdict_from_catalog(new.catalog) + old = d.get(new.key()) + if old is not None: + stmts.append(old.alter(new)) + else: + stmts.append(new.create_sql(self.dbconn.version)) + + # Check if the object just created was renamed, in which case + # don't try to delete the original one + if getattr(new, 'oldname', None): + try: + origname, new.name = new.name, new.oldname + oldkey = new.key() + finally: + new.name = origname + # Intentionally raising KeyError as tested e.g. in + # test_bad_rename_view -- ok Joe? + old = d[oldkey] + old._nodrop = True + + # Order the old database objects in reverse dependency order + old_objs = [] + for _, d in self.db.all_dicts(): + pairs = list(d.items()) + pairs.sort + old_objs.extend(list(map(itemgetter(1), pairs))) + old_objs = self.dep_sorted(old_objs, self.db) + old_objs.reverse() + + # Drop the objects that don't appear in the new db + for old in old_objs: + d = self.ndb.dbobjdict_from_catalog(old.catalog) + if isinstance(old, Table): + new = d.get(old.key()) + if new is not None: + stmts.extend(old.alter_drop_columns(new)) + if not getattr(old, '_nodrop', False) and old.key() not in d: + stmts.extend(old.drop()) + + if 'datacopy' in self.config: + opts.data_dir = self.config['files']['data_path'] + stmts.append(self.ndb.schemas.data_import(opts)) + + stmts = [s for s in flatten(stmts)] + funcs = False + for s in stmts: + if "LANGUAGE sql" in s and ( + s.startswith("CREATE FUNCTION ") or + s.startswith("CREATE OR REPLACE FUNCTION ")): + funcs = True + break + if funcs: + stmts.insert(0, "SET check_function_bodies = false") + + return stmts + + def dep_sorted(self, objs, db): + """Sort `objs` in order of dependency. + + The function implements the classic Kahn 62 algorighm, see + . + """ + # List of objects to return + L = [] + + # Collect the graph edges. + # Note that our "dependencies" are sort of backwards compared to the + # terms used in the algorithm (an edge in the algo would be from the + # schema to the table, we have the table depending on the schema) + ein = defaultdict(set) + eout = defaultdict(deque) + for obj in objs: + for dep in obj.get_deps(db): + eout[dep].append(obj) + ein[obj].add(dep) + + # The objects with no dependency to start with + S = deque() + for obj in objs: + if obj not in ein: + S.append(obj) + + while S: + # Objects with no dependencies can be emitted + obj = S.popleft() + L.append(obj) + + # Delete the edges and check if depending objects have no + # dependency now + while eout[obj]: + ch = eout[obj].popleft() + ein[ch].remove(obj) + if not ein[ch]: + del ein[ch] + S.append(ch) + + del eout[obj] # remove the empty set + + assert bool(ein) == bool(eout) + if not ein: + return L + else: + # is it possible? How do we deal with that? + raise Exception("the objects dependencies graph has loops") diff --git a/pyrseas/dbaugment.py b/pyrseas/dbaugment.py new file mode 100644 index 0000000..e655448 --- /dev/null +++ b/pyrseas/dbaugment.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""dbaugment - Augment a PostgreSQL database""" + +from __future__ import print_function +import sys +from argparse import FileType + +import yaml + +from pyrseas import __version__ +from pyrseas.yamlutil import yamldump +from pyrseas.augmentdb import AugmentDatabase +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(): + """Augment database specifications""" + parser = cmd_parser("Generate a modified schema for a PostgreSQL " + "database, in YAML format, augmented with specified " + "attributes and procedures", __version__) + # TODO: processing of multiple files, owner and privileges + parser.add_argument('-m', '--multiple-files', action='store_true', + help='multiple files (metadata directory)') + parser.add_argument('-O', '--no-owner', action='store_true', + help='exclude object ownership information') + parser.add_argument('-x', '--no-privileges', action='store_true', + dest='no_privs', + help='exclude privilege (GRANT/REVOKE) information') + parser.add_argument('spec', nargs='?', type=FileType('r'), + default=sys.stdin, help='YAML augmenter specification') + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + augdb = AugmentDatabase(cfg) + augmap = yaml.safe_load(options.spec) + try: + outmap = augdb.apply(augmap) + except BaseException as exc: + if type(exc) != KeyError: + raise + sys.exit("ERROR: %s" % str(exc)) + print(yamldump(outmap), file=output or sys.stdout) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/dbobject/__init__.py b/pyrseas/dbobject/__init__.py new file mode 100644 index 0000000..4abd59f --- /dev/null +++ b/pyrseas/dbobject/__init__.py @@ -0,0 +1,680 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject + ~~~~~~~~~~~~~~~~ + + This defines two low level classes and an intermediate class. + Most Pyrseas classes are derived from either DbObject or + DbObjectDict. +""" +import os +import re +import string +from functools import wraps + +from pyrseas.yamlutil import yamldump +from .privileges import privileges_to_map, add_grant, diff_privs +from .privileges import privileges_from_map + + +VALID_FIRST_CHARS = string.ascii_lowercase + '_' +VALID_CHARS = string.ascii_lowercase + string.digits + '_$' +RESERVED_WORDS = [] +NON_FILENAME_CHARS = re.compile(r'\W', re.U) +MAX_PG_IDENT_LEN = 63 +MAX_IDENT_LEN = int(os.environ.get("PYRSEAS_MAX_IDENT_LEN", 32)) + + +def fetch_reserved_words(db): + """Fetch PostgreSQL reserved words + + :param db: DbConnection object + """ + global RESERVED_WORDS + + if len(RESERVED_WORDS) == 0: + RESERVED_WORDS = [word["word"] for word in + db.fetchall("""SELECT word FROM pg_get_keywords() + WHERE catcode != 'U'""")] + + +def quote_id(name): + """Quotes an identifier if necessary. + + :param name: string to be quoted + + :return: possibly quoted string + """ + regular_id = True + if not name[0] in VALID_FIRST_CHARS or name in RESERVED_WORDS: + regular_id = False + else: + for ltr in name[1:]: + if ltr not in VALID_CHARS: + regular_id = False + break + + return regular_id and name or '"%s"' % name + + +def split_schema_obj(obj, sch=None): + """Return a (schema, object) tuple given a possibly schema-qualified name + + :param obj: object name or schema.object + :param sch: schema name (defaults to 'pg_catalog') + :return: tuple + """ + def undelim(ident): + if ident[0] == '"' and ident[-1] == '"': + ident = ident[1:-1] + return ident + + qualsch = sch + if sch is None: + qualsch = 'pg_catalog' + if obj[0] == '"' and obj[-1] == '"': + if '"."' in obj: + (qualsch, obj) = obj.split('"."') + qualsch = qualsch[1:] + obj = obj[:-1] + else: + obj = obj[1:-1] + else: + # TODO: properly handle functions + if '.' in obj and '(' not in obj: + (qualsch, obj) = obj.split('.') + if sch != qualsch: + sch = qualsch + return (undelim(sch), undelim(obj)) + + +def split_func_args(obj): + """Split function name and argument from a signature, e.g. fun(int, text) + + :param obj: The string to parse + :return: 2-item tuple (name, args), args is a list of strings. + + TODO: make it safer against pathologic input (names containing' '( and ',') + """ + tokens = obj.split('(') + if len(tokens) != 2 or not tokens[1].endswith(')'): + raise ValueError("not a valid function signature: '%s'" % obj) + name = tokens[0] + args = [arg.strip() for arg in tokens[1][:-1].split(',')] + return name, args + + +def commentable(func): + """Decorator to add comments to various objects""" + @wraps(func) + def add_comment(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if obj.description is not None: + stmts.append(obj.comment()) + return stmts + return add_comment + + +def grantable(func): + """Decorator to add GRANT to various objects""" + @wraps(func) + def grant(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if hasattr(obj, 'privileges'): + for priv in obj.privileges: + stmts.append(add_grant(obj, priv)) + return stmts + return grant + + +def ownable(func): + """Decorator to add ALTER OWNER to various objects""" + @wraps(func) + def add_alter(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if hasattr(obj, 'owner'): + stmts.append(obj.alter_owner()) + return stmts + return add_alter + + +class DbObject(object): + "A single object in a database catalog, e.g., a schema, a table, a column" + + keylist = ['name'] + """List of attributes that uniquely identify the object in the catalogs + + See description of :meth:`key` for further details. + """ + + @property + def objtype(self): + """Type of object as an uppercase string, for SQL syntax generation + + This is used in most CREATE, ALTER and DROP statements. It is + also used by :meth:`extern_key` in lowercase form. + """ + if self._objtype is None: + self._objtype = self.__class__.__name__.upper() + return self._objtype + + catalog = None + """The name of the system catalog where these objects live + """ + + allprivs = '' + + def __init__(self, name, description=None, **attrs): + """Initialize the catalog object from a dictionary of attributes + + :param name: name of object + :param description: comment text describing object + :param attrs: the dictionary of attributes + + Non-key attributes without a value are discarded. Values that + are multi-line strings are treated specially so that YAML will + output them in block style. + """ + self.name = name + self.description = description + self.depends_on = [] + self.owner = None + self.privileges = [] + self._objtype = None + + def _init_own_privs(self, owner=None, privileges=[]): + """Initialize owner and privileges attributes + + :param owner: name of user that owns the object +- :param privileges: privileges on object + + The vast majority of Postgres database objects have owner and + privileges attributes. Hence all base DbObject instances have + those attributes. This method allows separate initialization. + """ + self.owner = owner + if isinstance(privileges, str): + privileges = privileges.split(',') + self.privileges = privileges or [] + + def __repr__(self): + return "<%s at 0x%x>" % (self.extern_key(), id(self)) + + # hash and eq allow to use the objects as dict keys + def __hash__(self): + return hash((self.__class__, self.key())) + + def __eq__(self, other): + if self.__class__ is other.__class__: + return self.key() == other.key() + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def query(dbversion=None): + """The SQL SELECT query to fetch object instances from the catalogs + + :param dbversion: Postgres version identifier + + This is used by the method :meth:`fetch`. The `dbversion` + parameter is used in descendant classes to customize the + queries according to the target Postgres version. + """ + return "" + + def extern_key(self): + """Return the key to be used in external maps for this object + + :return: string + + This is used for the first two levels of external maps. The + first level is the one that includes schemas, as well as + extensions, languages, casts and foreign data wrappers. The + second level includes all schema-owned objects, i.e., tables, + functions, operators, etc. All subsequent levels, e.g., + primary keys, indexes, etc., currently use the object name as + the external identifier, appearing in the map after an object + grouping header, such as ``primary_key``. + + The common format for an external key is `object-type + non-schema-qualified-name`, where `object-type` is the + lowercase version of :attr:`objtype`, e.g., ``table + tablename``. Some object types require more, e.g., functions + need the signature, so they override this implementation. + + """ + return '%s %s' % (self.objtype.lower(), self.name) + + def extern_filename(self, ext='yaml', truncate=False): + """Return a filename to be used to output external files + + :param ext: file extension + :param truncate: truncate filename to MAX_IDENT_LEN + :return: filename string + + This is used for the first two levels of external (metadata) + files. The first level is the one that includes schemas, as + well as extensions, languages, casts and FDWs. The second + level includes all schema-owned objects, i.e., tables, + functions, operators, etc. + + The common format for the filename is `objtype.objname.yaml`, + e.g., for a table `t1` the filename is "table.t1.yaml". For + an object name that has characters not allowed in filesystems, + the characters are replaced by underscores. + + """ + max_len = MAX_IDENT_LEN if truncate else MAX_PG_IDENT_LEN + + def xfrm_filename(objtype, objid=None): + """Generic transformation of object identifier to a filename + + :param objtype: object type + :param objid: object identifier, usually the 'name' attribute + :return: filename string + """ + if objid: + filename = '%s.%.*s.%s' % ( + objtype, max_len, re.sub(NON_FILENAME_CHARS, '_', objid), + ext) + else: + filename = '%s.%s' % (objtype.replace(' ', '_'), ext) + return filename.lower() + + if hasattr(self, 'single_extern_file') and self.single_extern_file: + return xfrm_filename(self.objtype) + + return xfrm_filename(self.objtype, self.name) + + def key(self): + """Return a tuple that identifies the database object + + :return: a single string or a tuple of strings + + This is used as key for all internal maps. The first-level + objects (schemas, languages and casts) use the object name as + the key. Second-level (schema-owned) objects usually use the + schema name and the object name as the key. Some object types + need longer keys, e.g., operators need schema name, operator + symbols, left argument and right argument. + + Each class implementing an object type specifies a + :attr:`keylist` attribute, i.e., a list giving the names of + attributes making up the key. + """ + lst = [getattr(self, k) for k in self.keylist] + return len(lst) == 1 and lst[0] or tuple(lst) + + def identifier(self): + """Returns a full identifier for the database object + + :return: string + + This is used by :meth:`comment`, :meth:`alter_owner` and + :meth:`drop` to generate SQL syntax referring to the object. + It does not include the object type, but it may include (in + overridden methods) other elements, e.g., the arguments to a + function. + """ + return quote_id(self.__dict__[self.keylist[0]]) + + def to_map(self, db, no_owner=False, no_privs=False, deepcopy=True): + """Convert an object to a YAML-suitable format + + :param db: db used to tie the objects together + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + + The return value, a Python dictionary, is equivalent to a YAML + or JSON object. + """ + import copy + if deepcopy: + dct = copy.deepcopy(self.__dict__) + else: + dct = self.__dict__.copy() + for key in self.keylist: + del dct[key] + if self.description is None: + del dct['description'] + if no_owner or self.owner is None: + del dct['owner'] + if len(self.privileges) == 0 or no_privs: + del dct['privileges'] + else: + dct['privileges'] = self.map_privs() + + # Never dump the oid + dct.pop('oid', None) + + # Only dump dependencies that can't be inferred from the context + deps = set(dct.pop('depends_on', ())) + deps -= self.get_implied_deps(db) + if deps: + dct['depends_on'] = sorted([dep.extern_key() for dep in deps]) + + # Drop any private attributes + for k in list(dct.keys()): + if k.startswith('_'): + del dct[k] + + return dct + + def map_privs(self): + """Return a list of access privileges on the current object + + :return: list + """ + privlist = [] + for prv in self.privileges: + if prv: + privlist.append(privileges_to_map(prv, self.allprivs, + self.owner)) + sorted_privlist = [] + for sortedItem in sorted([list(i.keys())[0] for i in privlist]): + sorted_privlist.append([item for item in privlist + if list(item.keys())[0] == sortedItem][0]) + return sorted_privlist + + def set_oldname(self, inobj): + """Set oldname attribute if present in the input YAML map + + :param inobj: YAML map of input object + """ + if 'oldname' in inobj: + self.oldname = inobj.get('oldname') + + def fix_privileges(self): + """Adjust raw privilege information from YAML map""" + if len(self.privileges) > 0: + if self.owner is None: + raise ValueError( + "%s '%s' has privileges but no owner information" % ( + self.objtype.capitalize(), self.name)) + else: + self.privileges = privileges_from_map( + self.privileges, self.allprivs, self.owner) + + def _comment_text(self): + """Return the text for the SQL COMMENT statement + + :return: string + """ + if self.description is not None: + return "'%s'" % self.description.replace("'", "''") + else: + return 'NULL' + + def comment(self): + """Return SQL statement to create a COMMENT on the object + + :return: SQL statement + """ + return "COMMENT ON %s %s IS %s" % ( + self.objtype, self.identifier(), self._comment_text()) + + def alter_owner(self, owner=None): + """Return ALTER statement to set the OWNER of an object + + :return: SQL statement + """ + if self.owner != owner: + return "ALTER %s %s OWNER TO %s" % ( + self.objtype, self.identifier(), owner or self.owner) + else: + return [] + + def rename(self, oldname): + """Return SQL statement to RENAME the object + + :param oldname: the old name for the object + :return: SQL statement + """ + return "ALTER %s %s RENAME TO %s" % ( + self.objtype, quote_id(oldname), quote_id(self.name)) + + def create(self, dbversion=None): + raise NotImplementedError + + def create_sql(self, dbversion=None): + if hasattr(self, 'oldname') and self.oldname is not None: + return self.rename(self.oldname) + else: + return self.create(dbversion) + + def alter(self, inobj, no_owner=False): + """Generate SQL to transform an existing database object + + :param inobj: a YAML map defining the new object + :return: list of SQL statements + + Compares the current object to an input object and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if not no_owner and self.owner is not None and inobj.owner is not None: + if inobj.owner != self.owner: + stmts.append(self.alter_owner(inobj.owner)) + stmts.append(self.diff_privileges(inobj)) + stmts.append(self.diff_description(inobj)) + return stmts + + def drop(self): + """Generate SQL to drop the current object + + :return: list of SQL statements + """ + return ["DROP %s %s" % (self.objtype, self.identifier())] + + def diff_privileges(self, inobj): + """Generate SQL statements to grant or revoke privileges + + :param inobj: a YAML map defining the input object + :return: list of SQL statements + """ + return diff_privs(self, self.privileges, inobj, inobj.privileges) + + def diff_description(self, inobj): + """Generate SQL statements to add or change COMMENTs + + :param inobj: a YAML map defining the input object + :return: list of SQL statements + """ + stmts = [] + if self.description is not None: + if inobj.description is not None: + if self.description != inobj.description: + self.description = inobj.description + stmts.append(self.comment()) + else: + self.description = None + stmts.append(self.comment()) + else: + if inobj.description is not None: + self.description = inobj.description + stmts.append(self.comment()) + return stmts + + def get_deps(self, db): + """Return all the objects the object depends on + + The base implementation returns the explicit dependencies. Subclasses + may extend this to include implicit ones, which are implied e.g. by + containment in the yaml (such as an object on the schema they are on, a + constraint on the domain it is defined for. + + :return: set of `DbObject` + """ + deps = set() + + # The explicit dependencies + for dep in self.depends_on: + if isinstance(dep, str): + dep = db._get_by_extkey(dep) + deps.add(dep) + + for dep in self.get_implied_deps(db): + deps.add(dep) + + return deps + + def get_implied_deps(self, db): + """Return the dependencies the object can handle without being explicit + + :return: set of `DbObject` + """ + return set() + + +class DbSchemaObject(DbObject): + "A database object that is owned by a certain schema" + + def __init__(self, name, schema='public', description=None, **attrs): + super(DbSchemaObject, self).__init__(name, description, **attrs) + self.schema = schema + + def identifier(self): + """Return a full identifier for a schema object + + :return: string + """ + return "%s.%s" % (quote_id(self.schema), quote_id(self.name)) + + def qualname(self, schema=None, objname=None): + """Return the schema-qualified name of self or a related object + + :return: string + """ + if self.schema == schema and self.name == objname: + return self.identifier() + if objname is None: + objname = self.name + return "%s.%s" % (quote_id(schema or self.schema), quote_id(objname)) + + def unqualify(self, objname): + """Adjust the object name if it is qualified + + :param objname: object name + :return: unqualified object name + """ + if '.' in objname: + (sch, objname) = split_schema_obj(objname, self.schema) + assert sch == self.schema + return objname + else: + return objname + + def extern_filename(self, ext='yaml'): + """Return a filename to be used to output external files + + :param ext: file extension + :return: filename string + """ + return super(DbSchemaObject, self).extern_filename(ext, True) + + def rename(self, oldname): + """Return SQL statement to RENAME the schema object + + :param oldname: the old name for the schema object + :return: SQL statement + """ + return "ALTER %s %s.%s RENAME TO %s" % ( + self.objtype, quote_id(self.schema), quote_id(oldname), + quote_id(self.name)) + + def get_implied_deps(self, db): + deps = super(DbSchemaObject, self).get_implied_deps(db) + + # The schema of the object (if any) is always a dependency + if hasattr(self, 'schema'): + s = db.schemas.get(self.schema) + if s: + deps.add(s) + + return deps + + +class DbObjectDict(dict): + """A dictionary of database objects, all of the same type. + + However, note that "type" sometimes refers to a polymorphic class. + For example, a :class:`ConstraintDict` holds objects of type + :class:`Constraint`, but the actual objects may be of class + :class:`CheckConstraint`, :class:`PrimaryKey`, etc. + """ + + cls = DbObject + """The possibly-polymorphic class, derived from :class:`DbObject` that + the objects belong to. + """ + + def __init__(self, dbconn=None): + """Initialize the dictionary + + :param dbconn: a DbConnection object + + If dbconn is not None, the _from_catalog method is called to + initialize the dictionary from the catalogs. + """ + dict.__init__(self) + self.by_oid = {} + self.dbconn = dbconn + if dbconn: + self._from_catalog() + + def _from_catalog(self): + """Initialize the dictionary by querying the catalogs + + This is may be overridden by derived classes as needed. + """ + for obj in self.fetch(): + if hasattr(obj, 'options'): + if type(obj.options) is list: + obj.options = sorted(obj.options) + self[obj.key()] = obj + if hasattr(obj, 'oid'): + self.by_oid[obj.oid] = obj + + def to_map(self, db, opts): + """Convert the object dictionary to a regular dictionary + + :param db: db used to tie the objects together + :param opts: options to include/exclude information, etc. + :return: dictionary + + Invokes the `to_map` method of each object to construct the + dictionary. If `opts` specifies a directory, the objects are + written to files in that directory. + """ + objdict = {} + for objkey in sorted(self.keys()): + obj = self[objkey] + objmap = obj.to_map(db, opts.no_owner, opts.no_privs) + if objmap is not None: + extkey = obj.extern_key() + outobj = {extkey: objmap} + if opts.multiple_files: + filepath = obj.extern_filename() + with open(os.path.join(opts.metadata_dir, filepath), + 'a') as f: + f.write(yamldump(outobj)) + outobj = {extkey: filepath} + objdict.update(outobj) + return objdict + + def fetch(self): + """Fetch all objects from the catalogs using the associated + :meth:`query` methods. + + :return: list of self.cls (polymorphic) objects + + """ + self.query = self.cls.query(self.dbconn.version) + data = self.dbconn.fetchall(self.query) + self.dbconn.rollback() + return [self.cls(**dict(row)) for row in data] diff --git a/pyrseas/dbobject/cast.py b/pyrseas/dbobject/cast.py new file mode 100644 index 0000000..1565f60 --- /dev/null +++ b/pyrseas/dbobject/cast.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.cast + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Cast derived from DbObject and + CastDict derived from DbObjectDict. +""" +from . import DbObject, DbObjectDict, commentable +from . import split_func_args + + +CONTEXTS = {'a': 'assignment', 'e': 'explicit', 'i': 'implicit'} +METHODS = {'f': 'function', 'i': 'inout', 'b': 'binary coercible'} + + +class Cast(DbObject): + """A cast from a source type to a target type""" + + keylist = ['source', 'target'] + single_extern_file = True + catalog = 'pg_cast' + + def __init__(self, source, target, description, function, context, method, + oid=None): + """Initialize the cast + + :param source: source data type (from castsource) + :param target: target data type (from casttarget) + :param description: comment text (from obj_description()) + :param function: function to perform the cast (from castfunc) + :param context: context indicator (from castcontext) + :param method: method indicator (from castmethod) + """ + super(Cast, self).__init__('%s AS %s' % (source, target), description) + self._init_own_privs(None, []) + self.source = source + self.target = target + self.function = function + if len(context) == 1: + self.context = CONTEXTS[context] + else: + self.context = context + if len(method) == 1: + self.method = METHODS[method] + else: + self.method = method + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT castsource::regtype AS source, + casttarget::regtype AS target, + CASE WHEN castmethod = 'f' THEN castfunc::regprocedure + ELSE NULL::regprocedure END AS function, + castcontext AS context, castmethod AS method, + obj_description(c.oid, 'pg_cast') AS description, c.oid + FROM pg_cast c + JOIN pg_type s ON (castsource = s.oid) + JOIN pg_namespace sn ON (s.typnamespace = sn.oid) + JOIN pg_type t ON (casttarget = t.oid) + JOIN pg_namespace tn ON (t.typnamespace = tn.oid) + LEFT JOIN pg_proc p ON (castfunc = p.oid) + LEFT JOIN pg_namespace pn ON (p.pronamespace = pn.oid) + WHERE substring(sn.nspname for 3) != 'pg_' + OR substring(tn.nspname for 3) != 'pg_' + OR (castfunc != 0 AND substring(pn.nspname for 3) != 'pg_') + ORDER BY castsource, casttarget""" + + @staticmethod + def from_map(source, target, inobj): + """Initialize a Cast instance from a YAML map + + :param source: source type name + :param target: target type name + :param inobj: YAML map of the cast + :return: cast instance + """ + return Cast( + source, target, inobj.pop('description', None), + inobj.pop('function', None), inobj.get('context'), + inobj.get('method')) + + def extern_key(self): + """Return the key to be used in external maps for this cast + + :return: string + """ + return '%s (%s as %s)' % (self.objtype.lower(), self.source, + self.target) + + def identifier(self): + """Return a full identifier for a cast object + + :return: string + """ + return "(%s AS %s)" % (self.source, self.target) + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert a cast to a YAML-suitable format + + :return: dictionary + """ + dct = super(Cast, self).to_map(db) + del dct['name'] + if self.function is None: + del dct['function'] + return dct + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the cast + + :return: SQL statements + """ + with_clause = "\n WITH" + if self.function is not None: + with_clause += " FUNCTION %s" % self.function + elif self.method == 'inout': + with_clause += " INOUT" + else: + with_clause += "OUT FUNCTION" + as_clause = '' + if self.context == 'assignment': + as_clause = "\n AS ASSIGNMENT" + elif self.context == 'implicit': + as_clause = "\n AS IMPLICIT" + return ["CREATE CAST (%s AS %s)%s%s" % ( + self.source, self.target, with_clause, as_clause)] + + def get_implied_deps(self, db): + deps = super(Cast, self).get_implied_deps(db) + + # Types may be not found because they can be builtins + source = db.find_type(self.source) + if source: + deps.add(source) + + target = db.find_type(self.target) + if target: + deps.add(target) + + # The function instead we expect it exists + if self.method == 'function': + f = db.functions.find(*split_func_args(self.function)) + if f is not None: + deps.add(f) + + return deps + + +class CastDict(DbObjectDict): + "The collection of casts in a database" + + cls = Cast + + def from_map(self, incasts, newdb): + """Initialize the dictionary of casts by converting the input map + + :param incasts: YAML map defining the casts + :param newdb: collection of dictionaries defining the database + """ + for key in incasts: + if not key.startswith('cast (') or ' AS ' not in key.upper() \ + or key[-1:] != ')': + raise KeyError("Unrecognized object type: %s" % key) + asloc = key.upper().find(' AS ') + src = key[6:asloc] + trg = key[asloc + 4:-1] + inobj = incasts[key] + if not inobj: + raise ValueError("Cast '%s' has no specification" % key[5:]) + self[(src, trg)] = Cast.from_map(src, trg, inobj) diff --git a/pyrseas/dbobject/collation.py b/pyrseas/dbobject/collation.py new file mode 100644 index 0000000..c07c17b --- /dev/null +++ b/pyrseas/dbobject/collation.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.collation + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Collation and CollationDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable + + +class Collation(DbSchemaObject): + """A collation definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_collation' + + def __init__(self, name, schema, description, owner, lc_collate, lc_ctype, + oid=None): + """Initialize the collation + + :param name: collation name (from collname) + :param description: comment text (from obj_description()) + :param schema: schema name (from colnamespace) + :param owner: owner name (from rolname via collowner) + :param lc_collate: LC_COLLATE (from collcollate) + :param lc_ctype: LC_CTYPE (from collctype) + """ + super(Collation, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.lc_collate = lc_collate + self.lc_ctype = lc_ctype + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, collname AS name, rolname AS owner, + collcollate AS lc_collate, collctype AS lc_ctype, + obj_description(c.oid, 'pg_collation') AS description, c.oid + FROM pg_collation c + JOIN pg_roles r ON (r.oid = collowner) + JOIN pg_namespace n ON (collnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, collname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Collation instance from a YAML map + + :param name: collation name + :param name: schema map + :param inobj: YAML map of the collation + :return: Collation instance + """ + obj = Collation( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('lc_collate', None), + inobj.pop('lc_ctype', None)) + obj.set_oldname(inobj) + return obj + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the collation + + :return: SQL statements + """ + return ["CREATE COLLATION %s (\n LC_COLLATE = '%s'," + "\n LC_CTYPE = '%s')" % ( + self.qualname(), self.lc_collate, self.lc_ctype)] + + +class CollationDict(DbObjectDict): + "The collection of collations in a database." + + cls = Collation + + def from_map(self, schema, inmap): + """Initialize the dictionary of collations by examining the input map + + :param schema: the schema owing the collations + :param inmap: the input YAML map defining the collations + """ + for key in inmap: + if not key.startswith('collation '): + raise KeyError("Unrecognized object type: %s" % key) + name = key[10:] + inobj = inmap[key] + self[(schema.name, name)] = Collation.from_map(name, schema, inobj) diff --git a/pyrseas/dbobject/column.py b/pyrseas/dbobject/column.py new file mode 100644 index 0000000..e004fd8 --- /dev/null +++ b/pyrseas/dbobject/column.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.column + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Column derived from + DbSchemaObject and ColumnDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject, quote_id +from .privileges import privileges_from_map, add_grant, diff_privs + + +IDENTITY_TYPES = {'a': 'always', 'd': 'by default'} + + +class Column(DbSchemaObject): + "A table column or attribute of a composite type" + + keylist = ['schema', 'table'] # plus attribute number + allprivs = 'arwx' + + def __init__(self, name, schema, table, number, type, description=None, + privileges=[], not_null=True, default=None, identity=None, + collation=None, statistics=None, inherited=False, + dropped=False): + """Initialize the column + + :param name: column/attribute name (from attname) + :param schema: schema name (from nspname attrelid/relnamespace) + :param table: table/composite type name (from relame via attrelid) + :param description: comment text (from obj_description()) + :param privileges: access privileges (from attacl) + :param number: attribute number (from attnum) + :param type: data type (from atttypid/atttypmod) + :param not_null: not null constraint (from attnotnull) + :param default: default value expression (from pg_attrdef.adbin) + :param identity: type of identity column (from attidentity) + :param collation: collation name (from collname via attcollation) + :param statistics: statistics detail level (from attstattarget) + :param inherited: inherited indicator (from attinhcount) + :param dropped: dropped indicator (from attisdropped) + """ + super(Column, self).__init__(name, schema, description) + self._init_own_privs(None, privileges) + self.table = table + self.number = number + self.type = type + self.not_null = not_null + self.default = default + if identity == '' or identity is None: + self.identity = None + elif identity is not None and len(identity) == 1: + self.identity = IDENTITY_TYPES[identity] + else: + self.identity = identity + assert self.identity is None or \ + self.identity in IDENTITY_TYPES.values() + self.collation = collation + self.statistics = statistics + self.inherited = inherited + self.dropped = dropped + self._table = None + self._type = None + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS table, attname AS name, + attnum AS number, format_type(atttypid, atttypmod) AS type, + attnotnull AS not_null, attinhcount > 0 AS inherited, + pg_get_expr(adbin, adrelid) AS default, + attidentity AS identity, attstattarget AS statistics, + collname AS collation, attisdropped AS dropped, + array_to_string(attacl, ',') AS privileges, + col_description(c.oid, attnum) AS description + FROM pg_attribute JOIN pg_class c ON (attrelid = c.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + LEFT JOIN pg_attrdef ON (attrelid = pg_attrdef.adrelid + AND attnum = pg_attrdef.adnum) + LEFT JOIN pg_collation l ON (attcollation = l.oid) + WHERE relkind in ('c', 'r', 'f', 'p', 'v', 'm') + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND attnum > 0 + ORDER BY nspname, relname, attnum""" + + @staticmethod + def from_map(name, table, num, inobj): + """Initialize a Column instance from a YAML map + + :param name: column name + :param table: table map + :param num: column number + :param inobj: YAML map of the column + :return: Column instance + """ + obj = Column( + name, table.schema, table.name, num, inobj.pop('type', None), + inobj.pop('description', None), inobj.pop('privileges', []), + inobj.pop('not_null', False), inobj.pop('default', None), + inobj.pop('identity', None), inobj.pop('collation', None), + inobj.pop('statistics', None), inobj.pop('inherited', False), + inobj.pop('dropped', False)) + obj.set_oldname(inobj) + if len(obj.privileges) > 0: + if table.owner is None: + raise ValueError("Column '%s.%s' has privileges but " + "no owner information" % (table.name, name)) + obj.privileges = privileges_from_map( + obj.privileges, obj.allprivs, table.owner) + return obj + + def to_map(self, db, no_privs): + """Convert a column to a YAML-suitable format + + :param no_privs: exclude privilege information + :return: dictionary + """ + if self.dropped: + return None + dct = super(Column, self).to_map(db, False, no_privs, deepcopy=False) + del dct['number'], dct['name'], dct['dropped'] + if not self.not_null: + dct.pop('not_null') + if self.default is None: + dct.pop('default') + if self.identity is None: + dct.pop('identity') + if self.collation is None or self.collation == 'default': + dct.pop('collation') + if not self.inherited: + dct.pop('inherited') + if self.statistics is None or self.statistics == -1: + dct.pop('statistics') + return {self.name: dct} + + def add(self): + """Return a string to specify the column in a CREATE or ALTER TABLE + + :return: partial SQL statement + """ + stmt = "%s %s" % (quote_id(self.name), self.type) + if self.not_null: + stmt += ' NOT NULL' + if self.default is not None: + stmt += ' DEFAULT ' + self.default + if self.identity is not None: + stmt += " GENERATED %s AS IDENTITY" % self.identity.upper() + stmt += " (%s)" % self._owner_seq.add_inline() + if self.collation is not None and self.collation != 'default': + stmt += ' COLLATE "%s"' % self.collation + return (stmt, '' if self.description is None else self.comment()) + + def add_privs(self): + """Generate SQL statements to grant privileges on new column + + :return: list of SQL statements + """ + return [add_grant(self._table, priv, self.name) + for priv in self.privileges] + + def diff_privileges(self, incol): + """Generate SQL statements to grant or revoke privileges + + :param incol: a YAML map defining the input column + :return: list of SQL statements + """ + return [diff_privs(self._table, self.privileges, incol._table, + incol.privileges, self.name)] + + def comment(self): + """Return a SQL COMMENT statement for the column + + :return: SQL statement + """ + return "COMMENT ON COLUMN %s.%s IS %s" % ( + self._table.qualname(), self.name, self._comment_text()) + + def drop(self): + """Return string to drop the column via ALTER TABLE + + :return: SQL statement + """ + if self.dropped: + return [] + if self._table is not None: + (comptype, objtype) = (self._table.objtype, 'COLUMN') + compname = self._table.qualname() + elif self._type is not None: + (comptype, objtype) = ('TYPE', 'ATTRIBUTE') + compname = self._type.qualname() + else: + raise TypeError("Cannot determine type of %s", self.name) + return "ALTER %s %s DROP %s %s" % (comptype, compname, objtype, + quote_id(self.name)) + + def rename(self, newname): + """Return SQL statement to RENAME the column + + :param newname: the new name of the object + :return: SQL statement + """ + if self._table is not None: + (comptype, objtype) = (self._table.objtype, 'COLUMN') + compname = self._table.qualname() + elif self._type is not None: + (comptype, objtype) = ('TYPE', 'ATTRIBUTE') + compname = self._type.qualname() + else: + raise TypeError("Cannot determine type of %s", self.name) + stmt = "ALTER %s %s RENAME %s %s TO %s" % ( + comptype, compname, objtype, self.name, newname) + self.name = newname + return stmt + + def alter(self, incol): + """Generate SQL to transform an existing column + + :param insequence: a YAML map defining the new column + :return: list of partial SQL statements + + Compares the column to an input column and generates partial + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + base = "ALTER COLUMN %s " % quote_id(self.name) + # check NOT NULL + if not self.not_null and incol.not_null: + stmts.append(base + "SET NOT NULL") + if self.not_null and not incol.not_null: + stmts.append(base + "DROP NOT NULL") + # check data types + if self.type is None: + raise ValueError("Column '%s' missing datatype" % self.name) + if incol.type is None: + raise ValueError("Input column '%s' missing datatype" % incol.name) + if self.type != incol.type: + # validate type conversion? + stmts.append(base + "TYPE %s" % incol.type) + # check DEFAULTs + if self.default is None and incol.default is not None: + stmts.append(base + "SET DEFAULT %s" % incol.default) + if self.default is not None: + if incol.default is None: + stmts.append(base + "DROP DEFAULT") + elif self.default != incol.default: + stmts.append(base + "SET DEFAULT %s" % incol.default) + # check STATISTICS + if self.statistics is not None: + if self.statistics == -1 and (incol.statistics is not None + and incol.statistics != -1): + stmts.append(base + "SET STATISTICS %d" % incol.statistics) + if self.statistics != -1 and (incol.statistics is None + or incol.statistics == -1): + stmts.append(base + "SET STATISTICS -1") + + return (", ".join(stmts), self.diff_description(incol)) + + +class ColumnDict(DbObjectDict): + "The collection of columns in tables in a database" + + cls = Column + + def _from_catalog(self): + """Initialize the dictionary of columns by querying the catalogs""" + for col in self.fetch(): + sch, tbl = col.key() + if (sch, tbl) not in self: + self[(sch, tbl)] = [] + self[(sch, tbl)].append(col) + + def from_map(self, table, incols): + """Initialize the dictionary of columns by converting the input list + + :param table: table or type owning the columns/attributes + :param incols: YAML list defining the columns + """ + if not incols: + raise ValueError("Table '%s' has no columns" % table.name) + cols = self[(table.schema, table.name)] = [] + + for (num, incol) in enumerate(incols): + for key in incol: + if isinstance(incol[key], dict): + inobj = incol[key] + else: + inobj = {'type': incol[key]} + cols.append(Column.from_map(key, table, num, inobj)) diff --git a/pyrseas/dbobject/constraint.py b/pyrseas/dbobject/constraint.py new file mode 100644 index 0000000..12f932c --- /dev/null +++ b/pyrseas/dbobject/constraint.py @@ -0,0 +1,782 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.constraint + ~~~~~~~~~~~~~~~~~~ + + This module defines six classes: Constraint derived from + DbSchemaObject, CheckConstraint, PrimaryKey, ForeignKey and + UniqueConstraint derived from Constraint, and ConstraintDict + derived from DbObjectDict. + + TODO: UniqueConstraint and PrimaryKey are nearly identical. + Perhaps the latter should inherit from the former. +""" +import copy + +from . import DbObjectDict, DbSchemaObject +from . import quote_id, split_schema_obj, commentable +from .index import Index + + +class Constraint(DbSchemaObject): + """A constraint definition, such as a primary key, foreign key or + unique constraint. This also covers check constraints on domains.""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_constraint' + + def __init__(self, name, schema, table, description): + """Initialize the constraint + + :param name: constraint name (from conname) + :param schema: schema name (from connamespace) + :param table: table/domain name (from conrelid/contypid) + :param description: comment text (from obj_description()) + """ + super(Constraint, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = self.unqualify(table) + + def key_columns(self): + """Return comma-separated list of key column names + + :return: string + """ + return ", ".join([quote_id(col) for col in self.columns]) + + def _normalize_columns(self): + "Replace integer column numbers by column names" + if isinstance(self.columns[0], int): + self.columns = [self._table.columns[k - 1].name + for k in self.columns] + + def create(self, dbversion=None): + # TODO: is add really needed? + return self.add() + + @commentable + def add(self): + """Return string to add the constraint via ALTER TABLE + + :return: SQL statement + + Works as is for primary keys and unique constraints but has + to be overridden for check constraints and foreign keys. + """ + stmts = [] + tblspc = '' + if self.tablespace is not None: + tblspc = " USING INDEX TABLESPACE %s" % self.tablespace + stmts.append("ALTER TABLE %s ADD CONSTRAINT %s %s (%s)%s" % ( + self._table.qualname(), quote_id(self.name), + self.objtype, self.key_columns(), tblspc)) + if self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + return stmts + + def drop(self): + """Return string to drop the constraint via ALTER TABLE + + :return: SQL statement + """ + return ["ALTER %s %s DROP CONSTRAINT %s" % ( + self._table.objtype, self._table.qualname(), quote_id(self.name))] + + def comment(self): + """Return SQL statement to create COMMENT on constraint + + :return: SQL statement + """ + return "COMMENT ON CONSTRAINT %s ON %s IS %s" % ( + quote_id(self.name), self._table.qualname(), self._comment_text()) + + def get_implied_deps(self, db): + from .table import Table + from .dbtype import Domain + deps = super(Constraint, self).get_implied_deps(db) + + if isinstance(self._table, Table): + deps.add(db.tables[self.schema, self.table]) + elif isinstance(self._table, Domain): + deps.add(db.types[self.schema, self.table]) + else: + raise KeyError("Constraint '%s.%s' on unknown type/class" % ( + self.schema, self.name)) + + return deps + + +class CheckConstraint(Constraint): + "A check constraint definition" + + def __init__(self, name, schema, table, description, columns, + expression, is_domain_check=False, inherited=False, + oid=None): + """Initialize the check constraint + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param expression: constraint expression (from consrc) + :param is_domain_check: is constraint for a domain? (from contypid) + :param inherited: is it inherited? (from coninhcount) + """ + super(CheckConstraint, self).__init__(name, schema, table, description) + self.columns = columns + if expression[0] == '(': + assert expression[-1] == ')' + self.expression = expression + self.is_domain_check = is_domain_check + self.inherited = inherited + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + CASE WHEN contypid = 0 THEN conrelid::regclass::text + ELSE contypid::regtype::text END AS table, + contypid != 0 AS is_domain_check, conkey AS columns, + pg_get_expr(conbin, conrelid) AS expression, + coninhcount > 0 AS inherited, c.oid, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'c' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, target, inobj): + """Initialize a CheckConstraint instance from a YAML map + + :param name: constraint name + :param table: table map + :param target: column (default) or domain indicator + :param inobj: YAML map of the constraint + :return: CheckConstraint instance + """ + obj = CheckConstraint( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('expression', None), + (target != ''), inobj.pop('inherited', False)) + if 'depends_on' in inobj: + obj.depends_on.extend(inobj.pop('depends_on')) + return obj + + @property + def objtype(self): + return "CHECK" + + def to_map(self, db, dbcols): + """Convert a check constraint definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + dct = super(CheckConstraint, self).to_map(db, deepcopy=False) + dct.pop('is_domain_check') + if not self.inherited: + dct.pop('inherited') + if dbcols is not None and self.columns is not None: + dct['columns'] = [dbcols[k - 1] for k in self.columns] + else: + dct.pop('columns') + return {self.name: copy.deepcopy(dct)} + + @commentable + def add(self): + """Return string to add the CHECK constraint via ALTER TABLE + + :return: SQL statement + """ + # Don't generate inherited constraints + if self.inherited: + return [] + + if self.expression[0] != '(': + expr = "(%s)" % self.expression + else: + expr = self.expression + return ["ALTER %s %s ADD CONSTRAINT %s %s %s" % ( + self._table.objtype, self._table.qualname(), quote_id(self.name), + self.objtype, expr)] + + def drop(self): + if self.inherited: + return [] + else: + return super(CheckConstraint, self).drop() + + def alter(self, inchk): + """Generate SQL to transform an existing CHECK constraint + + :param inchk: a YAML map defining the new CHECK constraint + :return: list of SQL statements + + Compares the CHECK constraint to an input constraint and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + if inchk.expression != self.expression: + if inchk.expression.lower() != self.expression.lower(): + stmts.append(self.drop()) + stmts.append(inchk.add()) + stmts.append(self.diff_description(inchk)) + return stmts + + +class PrimaryKey(Constraint): + "A primary key constraint definition" + + def __init__(self, name, schema, table, description, columns, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the primary key + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is PK inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(PrimaryKey, self).__init__(name, schema, table, description) + self.columns = columns + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + amname AS access_method, spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl on (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'p' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a PrimaryKey instance from a YAML map + + :param name: key name + :param table: table map + :param inobj: YAML map of the primary key + :return: PrimaryKey instance + """ + return PrimaryKey( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + + @property + def objtype(self): + return "PRIMARY KEY" + + def to_map(self, db, dbcols): + """Convert a primary key definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + dct = super(PrimaryKey, self).to_map(db, deepcopy=False) + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.tablespace is None: + dct.pop('tablespace') + if '_table' in dct: + del dct['_table'] + dct['columns'] = [dbcols[k - 1] for k in self.columns] + return {self.name: copy.deepcopy(dct)} + + def alter(self, inpk): + """Generate SQL to transform an existing primary key + + :param inpk: a YAML map defining the new primary key + :return: list of SQL statements + + Compares the primary key to an input primary key and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + self._normalize_columns() + if inpk.columns != self.columns: + stmts.append(self.drop()) + stmts.append(inpk.add()) + if inpk.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self._table.qualname()) + stmts.append(self.diff_description(inpk)) + return stmts + + +ACTIONS = {'r': 'restrict', 'c': 'cascade', 'n': 'set null', + 'd': 'set default'} +MATCH_TYPES = {'f': 'full', 'p': 'partial', 's': 'simple'} + + +class ForeignKey(Constraint): + "A foreign key constraint definition" + + def __init__(self, name, schema, table, description, columns, + ref_table, ref_cols, on_update, on_delete, match, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the foreign key + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param ref_table: referenced table (from confrelid) + :param ref_cols: referenced columns (from confkey) + :param on_update: update action code (from confupdtype) + :param on_delete: delete action code (from confdeltype) + :param match: match action code (from confmatchtype) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is PK inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(ForeignKey, self).__init__(name, schema, table, description) + self.columns = columns + (self.ref_schema, self.ref_table) = split_schema_obj(ref_table, schema) + self.ref_cols = ref_cols + if on_update is not None and len(on_update) == 1: + self.on_update = None if on_update == 'a' else ACTIONS[on_update] + else: + assert on_update is None or on_update in ACTIONS.values() + self.on_update = on_update + if on_delete is not None and len(on_delete) == 1: + self.on_delete = None if on_delete == 'a' else ACTIONS[on_delete] + else: + assert on_delete is None or on_delete in ACTIONS.values() + self.on_delete = on_delete + if match is not None and len(match) == 1: + self.match = MATCH_TYPES[match] + else: + assert match is None or match in MATCH_TYPES.values() + self.match = 'simple' if match is None else match + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + confrelid::regclass AS ref_table, confkey AS ref_cols, + confupdtype AS on_update, confdeltype AS on_delete, + confmatchtype AS match, amname AS access_method, + spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl ON (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'f' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a ForeignKey instance from a YAML map + + :param name: key name + :param table: table map + :param inobj: YAML map of the foreign key + :return: ForeignKey instance + """ + if 'references' not in inobj: + raise KeyError("Constraint '%s' missing references" % name) + refs = inobj['references'] + if 'table' not in refs: + raise KeyError("Constraint '%s' missing table reference" % name) + ref_table = refs['table'] + if 'schema' in refs and refs['schema'] != table.schema: + ref_table = "%s.%s" % (refs['schema'], ref_table) + if 'columns' not in refs: + raise KeyError("Constraint '%s' missing reference columns" % name) + obj = ForeignKey( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), ref_table, refs['columns'], + inobj.pop('on_update', None), inobj.pop('on_delete', None), + inobj.pop('match', None), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + obj.depends_on.extend(inobj.get('depends_on', ())) + return obj + + @property + def objtype(self): + return "FOREIGN KEY" + + def _normalize_columns(self): + "Replace integer column numbers by column names" + super(ForeignKey, self)._normalize_columns() + if isinstance(self.ref_cols[0], int): + self.ref_cols = [self._references.columns[k - 1].name + for k in self.ref_cols] + + def ref_columns(self): + """Return comma-separated list of reference column names + + :return: string + """ + return ", ".join(self.ref_cols) + + def to_map(self, db, dbcols, refcols): + """Convert a foreign key definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + self._normalize_columns() + dct = super(ForeignKey, self).to_map(db, deepcopy=False) + if '_table' in dct: + del dct['_table'] + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + for attr in ('tablespace', 'on_update', 'on_delete'): + if getattr(self, attr) is None: + dct.pop(attr) + if self.match == 'simple': + dct.pop('match') + dct['references'] = {'table': dct['ref_table'], + 'columns': self.ref_cols} + if 'ref_schema' in dct: + dct['references'].update(schema=self.ref_schema) + dct.pop('ref_schema') + dct.pop('ref_table') + dct.pop('ref_cols') + + return {self.name: copy.deepcopy(dct)} + + @commentable + def add(self): + """Return string to add the foreign key via ALTER TABLE + + :return: SQL statement + """ + match = '' + if self.match is not None and self.match != 'simple': + match = " MATCH %s" % self.match.upper() + actions = '' + if self.on_update is not None: + actions = " ON UPDATE %s" % self.on_update.upper() + if self.on_delete is not None: + actions += " ON DELETE %s" % self.on_delete.upper() + if self.deferrable: + actions += " DEFERRABLE" + if self.deferred: + actions += " INITIALLY DEFERRED" + + return "ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) " \ + "REFERENCES %s (%s)%s%s" % ( + self._table.qualname(), quote_id(self.name), + self.key_columns(), self._references.qualname(), + self.ref_columns(), match, actions) + + def alter(self, infk): + """Generate SQL to transform an existing foreign key + + :param infk: a YAML map defining the new foreign key + :return: list of SQL statements + + Compares the foreign key to an input foreign key and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + self._normalize_columns() + if infk.columns != self.columns or infk.ref_cols != self.ref_cols \ + or infk.match != self.match or infk.on_update != self.on_update \ + or infk.on_delete != self.on_delete: + stmts.append(self.drop()) + stmts.append(infk.add()) + stmts.append(self.diff_description(infk)) + return stmts + + def get_implied_deps(self, db): + deps = super(ForeignKey, self).get_implied_deps(db) + + # add the table we reference + deps.add(self._references) + + # A fkey needs a pkey, unique constraint or complete unique index + # defined on the fields it references to be restored. + idx = self._find_referenced_index(db, self._references) + if idx: + deps.add(idx) + + return deps + + def _find_referenced_index(self, db, ref_table): + pkey = ref_table.primary_key + if pkey is not None and pkey.columns == self.ref_cols: + return pkey + + for uc in list(ref_table.unique_constraints.values()): + uc._normalize_columns() + if uc.columns == self.ref_cols: + return uc + + if hasattr(ref_table, 'indexes'): + if isinstance(self.ref_cols[0], int): + col_names = [ref_table.columns[i-1].name + for i in self.ref_cols] + else: + col_names = self.ref_cols + + for idx in list(ref_table.indexes.values()): + if getattr(idx, 'unique', False) \ + and not getattr(idx, 'predicate', None) \ + and idx.keys == col_names: + return idx + + +class UniqueConstraint(Constraint): + "A unique constraint definition" + + def __init__(self, name, schema, table, description, columns, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the unique constraint + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is it inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(UniqueConstraint, self).__init__( + name, schema, table, description) + self.columns = columns + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + amname AS access_method, spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl on (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'u' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a UniqueConstraint instance from a YAML map + + :param name: constraint name + :param table: table map + :param inobj: YAML map of the constraint + :return: UniqueConstraint instance + """ + return UniqueConstraint( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + + @property + def objtype(self): + return "UNIQUE" + + def to_map(self, db, dbcols): + """Convert a unique constraint definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + self._normalize_columns() + dct = super(UniqueConstraint, self).to_map(db, deepcopy=False) + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.tablespace is None: + dct.pop('tablespace') + return {self.name: copy.deepcopy(dct)} + + def alter(self, inuc): + """Generate SQL to transform an existing unique constraint + + :param inuc: a YAML map defining the new unique constraint + :return: list of SQL statements + + Compares the unique constraint to an input unique constraint + and generates SQL statements to transform it into the one + represented by the input. + """ + stmts = [] + self._normalize_columns() + if inuc.columns != self.columns: + stmts.append(self.drop()) + stmts.append(inuc.add()) + if inuc.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self._table.qualname()) + stmts.append(self.diff_description(inuc)) + return stmts + + +MATCHTYPES_PRE93 = {'f': 'full', 'p': 'partial', 'u': 'simple'} + + +class ConstraintDict(DbObjectDict): + "The collection of table or column constraints in a database" + + cls = Constraint + + def _from_catalog(self): + """Initialize the dictionary of constraints by querying the catalogs""" + for cls in (CheckConstraint, PrimaryKey, ForeignKey, + UniqueConstraint): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, table, inconstrs, target=''): + """Initialize the dictionary of constraints by converting the input map + + :param table: table affected by the constraints + :param inconstrs: YAML map defining the constraints + :param target: column or domain indicator + """ + if 'check_constraints' in inconstrs: + chks = inconstrs['check_constraints'] + for cns in chks: + inobj = chks[cns] + self[(table.schema, table.name, cns)] = \ + CheckConstraint.from_map(cns, table, target, inobj) + if 'primary_key' in inconstrs: + cns = list(inconstrs['primary_key'].keys())[0] + inobj = inconstrs['primary_key'][cns] + self[(table.schema, table.name, cns)] = PrimaryKey.from_map( + cns, table, inobj) + if 'foreign_keys' in inconstrs: + fkeys = inconstrs['foreign_keys'] + for cns in fkeys: + inobj = fkeys[cns] + self[(table.schema, table.name, cns)] = ForeignKey.from_map( + cns, table, inobj) + if 'unique_constraints' in inconstrs: + uconstrs = inconstrs['unique_constraints'] + for cns in uconstrs: + inobj = uconstrs[cns] + self[(table.schema, table.name, cns)] = \ + UniqueConstraint.from_map(cns, table, inobj) + + def link_refs(self, db): + for c in list(self.values()): + if isinstance(c, ForeignKey): + # The constraint depends on an index. Which one is accidental: + # it depends e.g. on which suitable index was available when + # the constraint was defined. So here we drop the dependency + # on the introspected one, while in get_implied_deps we give + # our best shot to suggest one to depend on. This way we don't + # need expliciting the dependency in yaml. + c.depends_on = [obj for obj in c.depends_on + if not isinstance(obj, Index)] + + if isinstance(c, (PrimaryKey, UniqueConstraint)): + # The constraint creates implicitly an index, so it depends on + # any extra dependencies the index has. These may include e.g. + # an operator class for a non-builtin type. + idx = db.indexes.get((c.schema, c.table, c.name)) + if idx: + c.depends_on.extend(idx.depends_on) diff --git a/pyrseas/dbobject/conversion.py b/pyrseas/dbobject/conversion.py new file mode 100644 index 0000000..32b1ae8 --- /dev/null +++ b/pyrseas/dbobject/conversion.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.conversion + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Conversion and ConversionDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable + + +class Conversion(DbSchemaObject): + """A conversion definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_conversion' + + def __init__(self, name, schema, description, owner, source_encoding, + dest_encoding, function, default=False, + oid=None): + """Initialize the conversion + + :param name: conversion name (from conname) + :param schema: schema name (from connamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via conowner) + :param source_encoding: source encoding (from conforencoding) + :param source_encoding: destination encoding (from contoencoding) + :param function: conversion function (from conproc) + :param default: indicates this is default conversion (from condefault) + """ + super(Conversion, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.source_encoding = source_encoding + self.dest_encoding = dest_encoding + self.function = function + self.default = default + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, conname AS name, rolname AS owner, + pg_encoding_to_char(c.conforencoding) AS source_encoding, + pg_encoding_to_char(c.contoencoding) AS dest_encoding, + conproc AS function, condefault AS default, c.oid, + obj_description(c.oid, 'pg_conversion') AS description + FROM pg_conversion c + JOIN pg_roles r ON (r.oid = conowner) + JOIN pg_namespace n ON (connamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, conname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Conversion instance from a YAML map + + :param name: conversion name + :param table: schema map + :param inobj: YAML map of the conversion + :return: Conversion instance + """ + obj = Conversion( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('source_encoding'), + inobj.pop('dest_encoding'), inobj.pop('function'), + inobj.pop('default', False)) + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert a conversion to a YAML-suitable format + + :return: dictionary + """ + dct = super(Conversion, self).to_map(db, no_owner) + if not self.default: + del dct['default'] + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the conversion + + :return: SQL statements + """ + dflt = '' + if self.default: + dflt = 'DEFAULT ' + return ["CREATE %sCONVERSION %s\n FOR '%s' TO '%s' FROM %s" % ( + dflt, self.qualname(), self.source_encoding, + self.dest_encoding, self.function)] + + +class ConversionDict(DbObjectDict): + "The collection of conversions in a database." + + cls = Conversion + + def from_map(self, schema, inmap): + """Initialize the dictionary of conversions by examining the input map + + :param schema: the schema owing the conversions + :param inmap: the input YAML map defining the conversions + """ + for key in inmap: + if not key.startswith('conversion '): + raise KeyError("Unrecognized object type: %s" % key) + cnv = key[11:] + inobj = inmap[key] + self[(schema.name, cnv)] = Conversion.from_map(cnv, schema, inobj) diff --git a/pyrseas/dbobject/dbtype.py b/pyrseas/dbobject/dbtype.py new file mode 100644 index 0000000..3420c46 --- /dev/null +++ b/pyrseas/dbobject/dbtype.py @@ -0,0 +1,740 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.dbtype + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines seven classes: DbType derived from + DbSchemaObject, BaseType, Composite, Domain, Enum and Range derived + from DbType, and TypeDict derived from DbObjectDict. +""" + +from . import DbObjectDict, DbSchemaObject +from . import split_schema_obj, commentable, ownable +from .constraint import CheckConstraint + +ALIGNMENT_TYPES = {'c': 'char', 's': 'int2', 'i': 'int4', 'd': 'double'} +STORAGE_TYPES = {'p': 'plain', 'e': 'external', 'm': 'main', 'x': 'extended'} + + +class DbType(DbSchemaObject): + """A user-defined type, such as a composite, domain or enum""" + + keylist = ['schema', 'name'] + catalog = 'pg_type' + + def __init__(self, name, schema, description, owner, privileges): + """Initialize the type + + :param name: type name (from typname) + :param schema: schema name (from typnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via typowner) + :param privileges: access privileges (from typacl) + """ + super(DbType, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + + @property + def objtype(self): + return "TYPE" + + def find_defining_funcs(self, dbfuncs): + return [] + + +class BaseType(DbType): + """A base type""" + + def __init__(self, name, schema, description, owner, privileges, + input, output, receive=None, send=None, typmod_in=None, + typmod_out=None, analyze=None, internallength=1, + alignment=None, storage=None, delimiter=',', + category=None, preferred=False, + oid=None): + """Initialize the base type + + :param name-privileges: see DbType.__init__ params + :param input: input function (see typinput) + :param output: output function (see typoutput) + :param receive: input conversion function (see typreceive) + :param send: output conversion function (see typsend) + :param typmod_in: type modifier input function (see typmodin) + :param typmod_out: type modifier output function (see typmodout) + :param analyze: custom ANALYZE function (see typanalyze) + :param internallength: length in bytes or -1 (variable) (see typlen) + :param alignment: storage alignment (see typalign) + :param storage: storage type for varlena types (see typstorage) + :param delimiter: delimiter character for array type (see typdelim) + :param category: PG data type classification (see typcategory) + :param preferred: preferred cast target? (see typispreferred) + """ + super(BaseType, self).__init__(name, schema, description, owner, + privileges) + self.input = self.unqualify(input) + self.output = self.unqualify(output) + self.receive = receive if receive != '-' else None + self.send = send if send != '-' else None + self.typmod_in = typmod_in if typmod_in != '-' else None + self.typmod_out = typmod_out if typmod_out != '-' else None + self.analyze = analyze if analyze != '-' else None + self.internallength = internallength + if alignment is not None and len(alignment) == 1: + self.alignment = ALIGNMENT_TYPES[alignment] + else: + assert alignment in ALIGNMENT_TYPES.values() + self.alignment = alignment + if storage is not None and len(storage) == 1: + self.storage = STORAGE_TYPES[storage] + else: + assert storage in STORAGE_TYPES.values() + self.storage = storage + self.delimiter = delimiter + self.category = category + self.preferred = preferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + typinput::regproc AS input, typoutput::regproc AS output, + typreceive::regproc AS receive, typsend::regproc AS send, + typmodin::regproc AS typmod_in, + typmodout::regproc AS typmod_out, + typanalyze::regproc AS analyze, + typlen AS internallength, typalign AS alignment, + typstorage AS storage, typdelim AS delimiter, + typcategory AS category, typispreferred AS preferred, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'b' AND typarray != 0 + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a BaseType instance from a YAML map + + :param name: BaseType name + :param schema: schema map + :param inobj: YAML map of the BaseType + :return: BaseType instance + """ + obj = BaseType( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('input', None), inobj.pop('output', None), + inobj.pop('receive', None), inobj.pop('send', None), + inobj.pop('typmod_in', None), inobj.pop('typmod_out', None), + inobj.pop('analyze', None), inobj.pop('internallength', 1), + inobj.pop('alignment', None), inobj.pop('storage', None), + inobj.pop('delimiter', ','), inobj.pop('category', None), + inobj.pop('preferred', False)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + dct = super(BaseType, self).to_map(db, no_owner, no_privs) + for attr in ('receive', 'send', 'typmod_in', 'typmod_out', 'analyze', + 'alignment', 'storage', 'category'): + if getattr(self, attr) is None: + dct.pop(attr) + if self.internallength < 0: + dct['internallength'] = 'variable' + if self.delimiter == ',': + dct.pop('delimiter') + if self.preferred is False: + dct.pop('preferred') + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the base type + + :return: SQL statements + """ + stmts = [] + opt_clauses = [] + if self.send is not None: + opt_clauses.append("SEND = %s" % self.send) + if self.receive is not None: + opt_clauses.append("RECEIVE = %s" % self.receive) + opt_clauses.append("INTERNALLENGTH = %s" % self.internallength) + if self.alignment is not None: + opt_clauses.append("ALIGNMENT = %s" % self.alignment) + if self.storage is not None: + opt_clauses.append("STORAGE = %s" % self.storage) + if self.delimiter is not None and self.delimiter != ',': + opt_clauses.append("DELIMITER = '%s'" % self.delimiter) + if self.category is not None: + opt_clauses.append("CATEGORY = '%s'" % self.category) + if self.preferred: + opt_clauses.append("PREFERRED = TRUE") + stmts.append("CREATE TYPE %s (\n INPUT = %s," + "\n OUTPUT = %s%s%s)" % ( + self.qualname(), self.input, self.output, + opt_clauses and ',\n ' or '', + ',\n '.join(opt_clauses))) + return stmts + + def get_implied_deps(self, db): + deps = super(BaseType, self).get_implied_deps(db) + for f in self.find_defining_funcs(db.functions): + deps.add(f) + + return deps + + def find_defining_funcs(self, dbfuncs): + rv = [] + for attr, arg in [ + ('input', 'cstring'), ('output', self.qualname()), + ('receive', 'internal'), ('send', self.qualname())]: + f = getattr(self, attr, None) + if not f: + continue + fschema, fname = split_schema_obj(f, self.schema) + rv.append(dbfuncs[fschema, fname, arg]) + return rv + + def drop(self): + """Generate SQL to drop the type (and related functions) + + :return: list of SQL statements + """ + # The CASCADE clause is required to also drop the related + # functions. There is a cyclic dependency so the dependency + # graph cannot be used. The functions will not be explicitly + # dropped. + return ["DROP %s %s CASCADE" % (self.objtype, self.identifier())] + + +class Composite(DbType): + """A composite type""" + + def __init__(self, name, schema, description, owner, privileges, + oid=None): + """Initialize the composite type + + :param name-privileges: see DbType.__init__ params + """ + super(Composite, self).__init__(name, schema, description, owner, + privileges) + self.attributes = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND (typtype = 'c' AND relkind = 'c') + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Composite instance from a YAML map + + :param name: Composite name + :param schema: schema map + :param inobj: YAML map of the Composite + :return: Composite instance + """ + obj = Composite( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + if len(self.attributes) == 0: + return + dct = super(Composite, self).to_map(db, no_owner, no_privs) + attrs = [] + for attr in self.attributes: + att = attr.to_map(db, False) + if att: + attrs.append(att) + dct['attributes'] = attrs + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the composite type + + :return: SQL statements + """ + attrs = [] + for att in self.attributes: + attrs.append(" " + att.add()[0]) + return ["CREATE TYPE %s AS (\n%s)" % ( + self.qualname(), ",\n".join(attrs))] + + def alter(self, intype): + """Generate SQL to transform an existing composite type + + :param intype: the new composite type + :return: list of SQL statements + + Compares the type to an input type and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if len(intype.attributes) == 0: + raise KeyError("Composite '%s' has no attributes" % intype.name) + attrnames = [attr.name for attr in self.attributes if not attr.dropped] + dbattrs = len(attrnames) + + base = "ALTER TYPE %s\n " % (self.qualname()) + # check input attributes + for (num, inattr) in enumerate(intype.attributes): + if hasattr(inattr, 'oldname'): + assert(self.attributes[num].name == inattr.oldname) + stmts.append(self.attributes[num].rename(inattr.name)) + # check existing attributes + if num < dbattrs and self.attributes[num].name == inattr.name: + (stmt, descr) = self.attributes[num].alter(inattr) + if stmt: + stmts.append(base + stmt) + if descr: + stmts.append(descr) + # add new attributes + elif inattr.name not in attrnames: + (stmt, descr) = inattr.add() + stmts.append(base + "ADD ATTRIBUTE %s" % stmt) + if descr: + stmts.append(descr) + + # Check the columns to drop + inattrnames = set(attr.name for attr in intype.attributes) + for attr in self.attributes: + if attr.name not in inattrnames: + stmts.append(attr.drop()) + + stmts.append(super(Composite, self).alter(intype)) + + return stmts + + def get_implied_deps(self, db): + deps = super(Composite, self).get_implied_deps(db) + for col in self.attributes: + type = db.find_type(col.type) + if type is not None: + deps.add(type) + + return deps + + +class Enum(DbType): + "An enumerated type definition" + + def __init__(self, name, schema, description, owner, privileges, + labels, oid=None): + """Initialize the enumerated type + + :param name-privileges: see DbType.__init__ params + """ + super(Enum, self).__init__(name, schema, description, owner, + privileges) + self.labels = labels + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + ARRAY(SELECT enumlabel FROM pg_enum e + WHERE t.oid = enumtypid + ORDER BY e.oid) AS labels, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'e' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize an Enum instance from a YAML map + + :param name: Enum name + :param schema: schema map + :param inobj: YAML map of the Enum + :return: Enum instance + """ + obj = Enum( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('labels', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the enum + + :return: SQL statements + """ + lbls = ["'%s'" % lbl for lbl in self.labels] + return ["CREATE TYPE %s AS ENUM (%s)" % ( + self.qualname(), ",\n ".join(lbls))] + + def alter(self, intype, no_owner=False): + """Generate SQL to transform an existing enum type + + :param intype: the new enum type + :return: list of SQL statements + + Compares the enum to an input enum and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if self.labels != intype.labels: + stmts.append(self.drop()) + stmts.append(intype.create()) + stmts.append(super(Enum, self).alter(intype, no_owner)) + return stmts + + +class Domain(DbType): + "A domain definition" + + def __init__(self, name, schema, description, owner, privileges, + type, not_null=False, default=None, + oid=None): + """Initialize the domain + + :param name-privileges: see DbType.__init__ params + :param type: type modifier (see typtypmod) + :param not_null: not null indicator (see typnotnull) + :param default: default value (see typdefault) + """ + super(Domain, self).__init__(name, schema, description, owner, + privileges) + self.type = type + self.not_null = not_null + self.default = default + self.check_constraints = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + format_type(typbasetype, typtypmod) AS type, + typnotnull AS not_null, typdefault AS default, + array_to_string(typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'd' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize an Domain instance from a YAML map + + :param name: Domain name + :param schema: schema map + :param inobj: YAML map of the Domain + :return: Domain instance + """ + obj = Domain( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('type', None), inobj.pop('not_null', False), + inobj.pop('default', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "DOMAIN" + + def to_map(self, db, no_owner, no_privs): + """Convert a domain to a YAML-suitable format + + :param no_owner: exclude domain owner information + :return: dictionary + """ + dct = super(Domain, self).to_map(db, no_owner, no_privs) + if self.not_null is False: + dct.pop('not_null') + if self.default is None: + dct.pop('default') + if len(self.check_constraints) > 0: + for cns in list(self.check_constraints.values()): + dct['check_constraints'].update( + self.check_constraints[cns.name].to_map(db, None)) + else: + dct.pop('check_constraints') + + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the domain + + :return: SQL statements + """ + create = "CREATE DOMAIN %s AS %s" % (self.qualname(), self.type) + if self.not_null: + create += ' NOT NULL' + if self.default is not None: + create += ' DEFAULT ' + str(self.default) + return [create] + + def get_implied_deps(self, db): + deps = super(Domain, self).get_implied_deps(db) + + # depend on the base type + # don't give errors in case it's a builtin + tschema, tname = split_schema_obj(self.type) + type = db.types.get((tschema, tname)) + if type: + deps.add(type) + + # In my testing database there is a dependency on the output + # function of the base type. TODO: investigate more. + if hasattr(type, 'output'): + fschema, fname = split_schema_obj(type.output) + func = db.functions[fschema, fname, type.qualname()] + deps.add(func) + + return deps + + +class Range(DbType): + "A range type definition" + + def __init__(self, name, schema, description, owner, privileges, + subtype, canonical=None, subtype_diff=None, + oid=None): + """Initialize the range type + + :param name-privileges: see DbType.__init__ params + :param subtype: type of range elements (from rngsubtype) + """ + super(Range, self).__init__(name, schema, description, owner, + privileges) + self.subtype = subtype + self.canonical = canonical if canonical != '-' else None + self.subtype_diff = subtype_diff if subtype_diff != '-' else None + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, t.typname AS name, rolname AS owner, + st.typname AS subtype, rn.rngcanonical AS canonical, + rn.rngsubdiff AS subtype_diff, + array_to_string(t.typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_range rn ON rngtypid = t.oid + JOIN pg_type st ON rngsubtype = st.oid + JOIN pg_roles r ON (r.oid = t.typowner) + JOIN pg_namespace n ON (t.typnamespace = n.oid) + WHERE t.typisdefined AND t.typtype = 'r' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, t.typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Range instance from a YAML map + + :param name: Range name + :param schema: schema map + :param inobj: YAML map of the Range + :return: Range instance + """ + obj = Range( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('subtype', None), inobj.pop('canonical', None), + inobj.pop('subtype_diff', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a range type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + dct = super(Range, self).to_map(db, no_owner, no_privs) + for attr in ('canonical', 'subtype_diff'): + if getattr(self, attr) is None: + dct.pop(attr) + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the range + + :return: SQL statements + """ + clauses = [] + if self.canonical is not None: + clauses.append("CANONICAL = %s" % self.canonical) + if self.subtype_diff is not None: + clauses.append("SUBTYPE_DIFF = %s" % self.subtype_diff) + return ["CREATE TYPE %s AS RANGE (SUBTYPE = %s%s%s)" % ( + self.qualname(), self.subtype, + clauses and ",\n " or "", ",\n ".join(clauses))] + + def alter(self, intype, no_owner=False): + """Generate SQL to transform an existing range type + + :param intype: the new range type + :return: list of SQL statements + + Compares the range to an input range and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + stmts.append(super(Range, self).alter(intype, no_owner)) + return stmts + + +class TypeDict(DbObjectDict): + "The collection of user-defined types in a database" + + cls = DbType + # TODO: consider to fetch all the objects belonging to extensions: + # not to dump them but to trace dependency from objects to the extension + + def _from_catalog(self): + """Initialize the dictionary of types by querying the catalogs""" + for cls in (BaseType, Composite, Domain, Enum, Range): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of types by converting the input map + + :param schema: schema owning the types + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + for k in inobjs: + (objtype, spc, key) = k.partition(' ') + if spc != ' ' or objtype not in ['domain', 'type']: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'domain': + inobj = inobjs[k] + self[(schema.name, key)] = obj = Domain.from_map( + key, schema, inobj) + newdb.constraints.from_map(obj, inobj, 'd') + elif objtype == 'type': + inobj = inobjs[k] + if 'input' in inobj: + self[(schema.name, key)] = BaseType.from_map( + key, schema, inobj) + elif 'attributes' in inobj: + self[(schema.name, key)] = obj = Composite.from_map( + key, schema, inobj) + try: + newdb.columns.from_map(obj, inobj['attributes']) + except KeyError as exc: + exc.args = ("Type '%s' has no attributes" % key, ) + raise + elif 'labels' in inobj: + self[(schema.name, key)] = obj = Enum.from_map( + key, schema, inobj) + elif 'subtype' in inobj: + self[(schema.name, key)] = obj = Range.from_map( + key, schema, inobj) + else: + raise KeyError("Unrecognized object type: %s" % k) + + def find(self, obj): + """Find a type given its name. + + The name can contain modifiers such as arrays '[]' and attributes '(3)' + + Return None if not found. + """ + schema, name = split_schema_obj(obj) + name = name.rstrip('[](,)0123456789') + return self.get((schema, name)) + + def link_refs(self, dbcolumns, dbconstrs, dbfuncs): + """Connect various objects to their corresponding types or domains + + :param dbcolumns: dictionary of columns + :param dbconstrs: dictionary of constraints + :param dbfuncs: dictionary of functions + + Fills the `check_constraints` dictionaries for each domain by + traversing the `dbconstrs` dictionary. Fills the attributes + list for composite types. Fills the dependent functions + dictionary for base types. + """ + for (sch, typ) in dbcolumns: + if (sch, typ) in self: + assert isinstance(self[(sch, typ)], Composite) + self[(sch, typ)].attributes = dbcolumns[(sch, typ)] + for attr in dbcolumns[(sch, typ)]: + attr._type = self[(sch, typ)] + for (sch, typ, cns) in dbconstrs: + constr = dbconstrs[(sch, typ, cns)] + if isinstance(constr, CheckConstraint) and \ + constr.is_domain_check: + constr._table = dbtype = self[(sch, typ)] + dbtype.check_constraints.update({cns: constr}) diff --git a/pyrseas/dbobject/eventtrig.py b/pyrseas/dbobject/eventtrig.py new file mode 100644 index 0000000..c52fece --- /dev/null +++ b/pyrseas/dbobject/eventtrig.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.eventtrig + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: EventTrigger derived from + DbObject, and EventTriggerDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable +from .function import split_schema_func, join_schema_func + +EXEC_PROC = 'EXECUTE PROCEDURE ' + +ENABLE_MODES = {'O': True, 'D': False, 'R': 'replica', 'A': 'always'} + + +class EventTrigger(DbObject): + """An event trigger""" + + keylist = ['name'] + catalog = 'pg_event_trigger' + + def __init__(self, name, description, owner, event, procedure, + enabled=False, tags=None, + oid=None): + """Initialize the event trigger + + :param name: trigger name (from evtname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via evtowner) + :param event: event that causes firing (from evtevent) + :param procedure: function to be called (from evtfoid) + :param enabled: replication mode firing control (from evtenabled) + :param tags: command tags (from evttags) + """ + super(EventTrigger, self).__init__(name, description) + self._init_own_privs(owner, []) + self.event = event + if procedure[-2:] == '()': + procedure = procedure[:-2] + self.procedure = split_schema_func(None, procedure) + if enabled is False or enabled is True: + self.enabled = enabled + elif len(enabled) == 1: + self.enabled = ENABLE_MODES[enabled] + else: + assert enabled is not None and enabled in ENABLE_MODES.values() + self.enabled = enabled + self.tags = tags + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT evtname AS name, evtevent AS event, rolname AS owner, + evtenabled AS enabled, evtfoid::regprocedure AS procedure, + evttags AS tags, t.oid, + obj_description(t.oid, 'pg_event_trigger') AS description + FROM pg_event_trigger t JOIN pg_roles ON (evtowner = pg_roles.oid) + WHERE t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e') + ORDER BY name""" + + @staticmethod + def from_map(name, inobj): + """Initialize an event trigger instance from a YAML map + + :param name: trigger name + :param inobj: YAML map of the event trigger + :return: event trigger instance + """ + obj = EventTrigger( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('event', None), inobj.pop('procedure', None), + inobj.pop('enabled', False), inobj.pop('tags', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "EVENT TRIGGER" + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert an event trigger definition to a YAML-suitable format + + :param db: db used to tie the objects together + :return: dictionary + """ + dct = super(EventTrigger, self).to_map(db, no_owner) + dct['procedure'] = join_schema_func(self.procedure) + "()" + if self.tags is None: + dct.pop('tags') + return dct + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the event trigger + + :return: SQL statements + """ + filter = '' + if self.tags is not None: + filter = "\n WHEN tag IN (%s)" % ", ".join( + ["'%s'" % tag for tag in self.tags]) + return ["CREATE %s %s\n ON %s%s\n EXECUTE PROCEDURE %s()" % ( + self.objtype, quote_id(self.name), self.event, filter, + join_schema_func(self.procedure))] + + def get_implied_deps(self, db): + deps = super(EventTrigger, self).get_implied_deps(db) + sch, fnc = self.procedure + deps.add(db.functions[(sch, fnc, '')]) + return deps + + +class EventTriggerDict(DbObjectDict): + "The collection of event triggers in a database" + + cls = EventTrigger + + def _from_catalog(self): + """Initialize the dictionary of triggers by querying the catalogs""" + if self.dbconn.version < 90300: + return + super(EventTriggerDict, self)._from_catalog() + + def from_map(self, intriggers, newdb): + """Initialize the dictionary of triggers by converting the input map + + :param intriggers: YAML map defining the event triggers + :param newdb: dictionary of input database + """ + for key in intriggers: + if not key.startswith('event trigger '): + raise KeyError("Unrecognized object type: %s" % key) + trg = key[14:] + inobj = intriggers[key] + if not inobj: + raise ValueError("Event trigger '%s' has no specification" % + trg) + self[trg] = EventTrigger.from_map(trg, inobj) diff --git a/pyrseas/dbobject/extension.py b/pyrseas/dbobject/extension.py new file mode 100644 index 0000000..8c4733f --- /dev/null +++ b/pyrseas/dbobject/extension.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.extension + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Extension derived from DbObject, + and ExtensionDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable + + +class Extension(DbObject): + """An extension""" + + keylist = ['name'] + single_extern_file = True + catalog = 'pg_extension' + + def __init__(self, name, description, owner, schema, version=None, + oid=None): + """Initialize the extension + + :param name: extension name (from extlname) + :param description: comment text (from obj_description()) + :param schema: schema name (from extnamespace) + :param owner: owner name (from rolname via extowner) + :param version: version name (from extversion) + """ + super(Extension, self).__init__(name, description) + self._init_own_privs(owner, []) + self.schema = schema + self.version = version + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT e.extname AS name, n.nspname AS schema, e.extversion AS version, + r.rolname AS owner, + obj_description(e.oid, 'pg_extension') AS description, e.oid + FROM pg_extension e + JOIN pg_roles r ON (r.oid = e.extowner) + JOIN pg_namespace n ON (e.extnamespace = n.oid) + WHERE n.nspname != 'information_schema' + ORDER BY e.extname""" + + @staticmethod + def from_map(name, inobj): + """Initialize an Extension instance from a YAML map + + :param name: extension name + :param inobj: YAML map of the extension + :return: extension instance + """ + return Extension( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.get('schema'), inobj.pop('version', None)) + + def get_implied_deps(self, db): + """Return the implied dependencies of the object + + :param db: the database where this object exists + :return: set of `DbObject` + """ + deps = super(Extension, self).get_implied_deps(db) + if self.schema is not None: + s = db.schemas.get(self.schema) + if s: + deps.add(s) + + return deps + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the extension + + :return: SQL statements + """ + opt_clauses = [] + if self.schema is not None and self.schema not in ( + 'pg_catalog', 'public'): + opt_clauses.append("SCHEMA %s" % quote_id(self.schema)) + if self.version is not None: + opt_clauses.append("VERSION '%s'" % self.version) + return ["CREATE EXTENSION %s%s" % ( + quote_id(self.name), ('\n ' + '\n '.join(opt_clauses)) + if opt_clauses else '')] + + def alter(self, inobj, no_owner=True): + """Generate SQL to transform an existing extension + + :param inobj: a YAML map defining the new extension + :return: list of SQL statements + + This exists because ALTER EXTENSION does not permit altering + the owner. + """ + return super(Extension, self).alter(inobj, no_owner=no_owner) + +CORE_LANGS = [ + "plpgsql", + "pltcl", + "pltclu", + "plperl", + "plperlu", + "plpythonu", + "plpython2u", + "plpython3u"] + +class ExtensionDict(DbObjectDict): + "The collection of extensions in a database" + + cls = Extension + + def _from_catalog(self): + """Initialize the dictionary of extensions by querying the catalogs""" + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, inexts, newdb): + """Initialize the dictionary of extensions by converting the input map + + :param inexts: YAML map defining the extensions + :param newdb: dictionary of input database + """ + for key in inexts: + if not key.startswith('extension '): + raise KeyError("Unrecognized object type: %s" % key) + name = key[10:] + inobj = inexts[key] + self[name] = Extension.from_map(name, inobj) + if self[name].name in CORE_LANGS: + lang = {'language %s' % self[name].name: { + '_ext': 'e', 'owner': self[name].owner}} + newdb.languages.from_map(lang) diff --git a/pyrseas/dbobject/foreign.py b/pyrseas/dbobject/foreign.py new file mode 100644 index 0000000..0da995a --- /dev/null +++ b/pyrseas/dbobject/foreign.py @@ -0,0 +1,693 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.foreign + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines nine classes: DbObjectWithOptions derived from + DbObject, ForeignDataWrapper, ForeignServer and UserMapping + derived from DbObjectWithOptions, ForeignDataWrapperDict, + ForeignServerDict and UserMappingDict derived from DbObjectDict, + ForeignTable derived from DbObjectWithOptions and Table, and + ForeignTableDict derived from ClassDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable, ownable, grantable +from .table import ClassDict, Table + + +class DbObjectWithOptions(DbObject): + """Helper class for database objects with OPTIONS clauses""" + + def __init__(self, name, options): + """Initialize the database object with options + + :param name: object name (from fdwname, srvname, etc.) + :param options: object specific options (from fdwoptions, etc.) + """ + super(DbObjectWithOptions, self).__init__(name, None) + self.options = {} if options is None else options + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert objects to a YAML-suitable format + + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(DbObjectWithOptions, self).to_map(db, no_owner, no_privs) + if len(self.options) == 0: + dct.pop('options') + return dct + + def options_clause(self): + """Create the OPTIONS clause + + :param optdict: the dictionary of options + :return: SQL OPTIONS clause + """ + opts = [] + for opt in self.options: + (nm, val) = opt.split('=', 1) + opts.append("%s '%s'" % (nm, val)) + return "OPTIONS (%s)" % ', '.join(opts) + + def diff_options(self, newopts): + """Compare options lists and generate SQL OPTIONS clause + + :newopts: list of new options + :return: SQL OPTIONS clause + + Generate ([ADD|SET|DROP key 'value') clauses from two lists in the + form of 'key=value' strings. + """ + def to_dict(optlist): + return dict(opt.split('=', 1) for opt in optlist) + + oldopts = {} + if len(self.options) > 0: + oldopts = to_dict(self.options) + newopts = to_dict(newopts) + clauses = [] + for key, val in list(newopts.items()): + if key not in oldopts: + clauses.append("%s '%s'" % (key, val)) + elif val != oldopts[key]: + clauses.append("SET %s '%s'" % (key, val)) + for key, val in list(oldopts.items()): + if key not in newopts: + clauses.append("DROP %s" % key) + return clauses and "OPTIONS (%s)" % ', '.join(clauses) or '' + + def alter(self, inobj): + """Generate SQL to transform an existing object with options + + :param inobj: a YAML map defining the new object + :return: list of SQL statements + """ + stmts = super(DbObjectWithOptions, self).alter(inobj) + newopts = [] + if len(inobj.options) > 0: + newopts = inobj.options + diff_opts = self.diff_options(newopts) + if diff_opts: + stmts.append("ALTER %s %s %s" % ( + self.objtype, self.identifier(), diff_opts)) + return stmts + + +class ForeignDataWrapper(DbObjectWithOptions): + """A foreign data wrapper definition""" + + single_extern_file = True + catalog = 'pg_foreign_data_wrapper' + + def __init__(self, name, options, description, owner, privileges, + handler=None, validator=None, + oid=None): + """Initialize the foreign data wrapper + + :param name-options: see DbObjectWithOptions.__init__ params + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via fdwowner) + :param privileges: access privileges (from fdwacl) + :param handler: handler function (from fdwhandler) + :param validator: validator function (from fdwvalidator) + """ + super(ForeignDataWrapper, self).__init__(name, options) + self.description = description + self._init_own_privs(owner, privileges) + self.handler = handler + self.validator = validator + self.servers = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS name, CASE WHEN fdwhandler = 0 THEN NULL + ELSE fdwhandler::regproc END AS handler, + CASE WHEN fdwvalidator = 0 THEN NULL + ELSE fdwvalidator::regproc END AS validator, + fdwoptions AS options, rolname AS owner, + array_to_string(fdwacl, ',') AS privileges, + obj_description(w.oid, 'pg_foreign_data_wrapper') AS + description, w.oid + FROM pg_foreign_data_wrapper w + JOIN pg_roles r ON (r.oid = fdwowner) + ORDER BY fdwname""" + + @staticmethod + def from_map(name, inobj): + """Initialize an FDW instance from a YAML map + + :param name: FDW name + :param inobj: YAML map of the FDW + :return: FDW instance + """ + obj = ForeignDataWrapper( + name, inobj.pop('options', {}), inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('handler', None), inobj.pop('validator', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "FOREIGN DATA WRAPPER" + + @property + def allprivs(self): + return 'U' + + def to_map(self, db, no_owner, no_privs): + """Convert wrappers and subsidiary objects to a YAML-suitable format + + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(ForeignDataWrapper, self).to_map(db, no_owner, no_privs) + for attr in ('handler', 'validator'): + if getattr(self, attr) is None: + dct.pop(attr) + srvs = {} + for srv in self.servers: + srvs.update(self.servers[srv].to_map(db, no_owner, no_privs)) + dct.update(srvs) + dct.pop('servers') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the data wrapper + + :return: SQL statements + """ + clauses = [] + for fnc in ('validator', 'handler'): + if getattr(self, fnc) is not None: + clauses.append("%s %s" % (fnc.upper(), getattr(self, fnc))) + if len(self.options) > 0: + clauses.append(self.options_clause()) + return ["CREATE FOREIGN DATA WRAPPER %s%s" % ( + quote_id(self.name), + clauses and '\n ' + ',\n '.join(clauses) or '')] + + +class ForeignDataWrapperDict(DbObjectDict): + "The collection of foreign data wrappers in a database" + + cls = ForeignDataWrapper + + def from_map(self, inwrappers, newdb): + """Initialize the dictionary of wrappers by examining the input map + + :param inwrappers: input YAML map defining the data wrappers + :param newdb: collection of dictionaries defining the database + """ + for key in inwrappers: + if not key.startswith('foreign data wrapper '): + raise KeyError("Unrecognized object type: %s" % key) + fdw = key[21:] + inobj = inwrappers[key] + inservs = {} + for key in inobj: + if key.startswith('server '): + inservs.update({key: inobj[key]}) + self[fdw] = ForeignDataWrapper.from_map(fdw, inobj) + newdb.servers.from_map(self[fdw], inservs, newdb) + + def link_refs(self, dbservers): + """Connect servers to their respective foreign data wrappers + + :param dbservers: dictionary of foreign servers + """ + for (fdw, srv) in dbservers: + dbserver = dbservers[(fdw, srv)] + assert self[fdw] + wrapper = self[fdw] + if not hasattr(wrapper, 'servers'): + wrapper.servers = {} + wrapper.servers.update({srv: dbserver}) + + +class ForeignServer(DbObjectWithOptions): + """A foreign server definition""" + + privobjtype = "FOREIGN SERVER" + keylist = ['wrapper', 'name'] + catalog = 'pg_foreign_server' + + def __init__(self, name, options, description, owner, privileges, + wrapper, type=None, version=None, + oid=None): + """Initialize the foreign server + + :param name-options: see DbObjectWithOptions.__init__ params + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via srvowner) + :param privileges: access privileges (from srvacl) + :param wrapper: foreign data wrapper (from fdwname via srvfdw) + :param type: server type (from srvtype) + :param version: version (from srvversion) + """ + super(ForeignServer, self).__init__(name, options) + self.description = description + self._init_own_privs(owner, privileges) + self.wrapper = wrapper + self.type = type + self.version = version + self.usermaps = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS wrapper, srvname AS name, srvtype AS type, + srvversion AS version, srvoptions AS options, + rolname AS owner, + array_to_string(srvacl, ',') AS privileges, s.oid, + obj_description(s.oid, 'pg_foreign_server') AS description + FROM pg_foreign_server s JOIN pg_roles r ON (r.oid = srvowner) + JOIN pg_foreign_data_wrapper w ON (srvfdw = w.oid) + ORDER BY fdwname, srvname""" + + @staticmethod + def from_map(name, wrapper, inobj): + """Initialize a foreign server instance from a YAML map + + :param name: server name + :param wrapper: FDW name + :param inobj: YAML map of the server + :return: foreign server instance + """ + obj = ForeignServer( + name, inobj.pop('options', {}), inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), wrapper, + inobj.pop('type', None), inobj.pop('version', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "SERVER" + + @property + def allprivs(self): + return 'U' + + def identifier(self): + """Returns a full identifier for the foreign server + + :return: string + """ + return quote_id(self.name) + + def to_map(self, db, no_owner, no_privs): + """Convert servers and subsidiary objects to a YAML-suitable format + + :param no_owner: exclude server owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(ForeignServer, self).to_map(db, no_owner, no_privs) + for attr in ('type', 'version'): + if getattr(self, attr) is None: + dct.pop(attr) + key = self.extern_key() + server = {key: dct} + server[key].pop('usermaps') + if len(self.usermaps) > 0: + umaps = {} + for umap in self.usermaps: + umaps.update({umap: self.usermaps[umap].to_map(db)}) + server[key]['user mappings'] = umaps + + return server + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the server + + :return: SQL statements + """ + clauses = [] + options = [] + for opt in ('type', 'version'): + if getattr(self, opt) is not None: + clauses.append("%s '%s'" % (opt.upper(), getattr(self, opt))) + if len(self.options) > 0: + options.append(self.options_clause()) + return ["CREATE SERVER %s%s\n FOREIGN DATA WRAPPER %s%s" % ( + quote_id(self.name), + clauses and ' ' + ' '.join(clauses) or '', + quote_id(self.wrapper), + options and '\n ' + ',\n '.join(options) or '')] + + def get_implied_deps(self, db): + deps = super(ForeignServer, self).get_implied_deps(db) + deps.add(db.fdwrappers[self.wrapper]) + return deps + + +class ForeignServerDict(DbObjectDict): + "The collection of foreign servers in a database" + + cls = ForeignServer + + def from_map(self, wrapper, inservers, newdb): + """Initialize the dictionary of servers by examining the input map + + :param wrapper: associated foreign data wrapper + :param inservers: input YAML map defining the foreign servers + :param newdb: collection of dictionaries defining the database + """ + for key in inservers: + if not key.startswith('server '): + raise KeyError("Unrecognized object type: %s" % key) + srv = key[7:] + inobj = inservers[key] + self[(wrapper.name, srv)] = serv = ForeignServer.from_map( + srv, wrapper.name, inobj) + if 'user mappings' in inobj: + newdb.usermaps.from_map(serv, inobj['user mappings']) + + def to_map(self, db, no_owner, no_privs): + """Convert the server dictionary to a regular dictionary + + :param no_owner: exclude server owner information + :param no_privs: exclude privilege information + :return: dictionary + + Invokes the `to_map` method of each server to construct a + dictionary of foreign servers. + """ + servers = {} + for srv in self: + servers.update(self[srv].to_map(db, no_owner, no_privs)) + return servers + + def link_refs(self, dbusermaps): + """Connect user mappings to their respective servers + + :param dbusermaps: dictionary of user mappings + """ + for (fdw, srv, usr) in dbusermaps: + dbusermap = dbusermaps[(fdw, srv, usr)] + assert self[(fdw, srv)] + server = self[(fdw, srv)] + server.usermaps.update({usr: dbusermap}) + + +class UserMapping(DbObjectWithOptions): + """A user mapping definition""" + + keylist = ['wrapper', 'server', 'name'] + catalog = 'pg_user_mappings' + + def __init__(self, name, options, wrapper, server, + oid=None): + """Initialize the user mapping + + :param name-options: see DbObjectWithOptions.__init__ params + :param wrapper: foreign data wrapper (from fdwname via srvfdw) + :param server: server name (from umserver) + :param version: version (from srvversion) + """ + super(UserMapping, self).__init__(name, options) + self.wrapper = wrapper + self.server = server + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS wrapper, s.srvname AS server, + CASE umuser WHEN 0 THEN 'PUBLIC' ELSE + usename END AS name, umoptions AS options, u.umid AS oid + FROM pg_user_mappings u + JOIN pg_foreign_server s ON (u.srvid = s.oid) + JOIN pg_foreign_data_wrapper w ON (srvfdw = w.oid) + ORDER BY fdwname, s.srvname, 3""" + + @staticmethod + def from_map(name, server, inobj): + """Initialize a user mapping instance from a YAML map + + :param name: mapping name + :param server: foreign server map + :param inobj: YAML map of the user mapping + :return: user mapping instance + """ + obj = UserMapping(name, inobj.pop('options', {}), server.wrapper, + server.name) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "USER MAPPING" + + def extern_key(self): + """Return the key to be used in external maps for this user mapping + + :return: string + """ + return self.name + + def identifier(self): + """Return a full identifier for a user mapping object + + :return: string + """ + return "FOR %s SERVER %s" % ( + self.name == 'PUBLIC' and 'PUBLIC' or quote_id(self.name), + quote_id(self.server)) + + def create(self, dbversion=None): + """Return SQL statements to CREATE the user mapping + + :return: SQL statements + """ + options = [] + if len(self.options) > 0: + options.append(self.options_clause()) + return ["CREATE USER MAPPING FOR %s\n SERVER %s%s" % ( + self.name == 'PUBLIC' and 'PUBLIC' or + quote_id(self.name), quote_id(self.server), + options and '\n ' + ',\n '.join(options) or '')] + + def get_implied_deps(self, db): + deps = super(UserMapping, self).get_implied_deps(db) + deps.add(db.fdwrappers[self.wrapper]) + return deps + + +class UserMappingDict(DbObjectDict): + "The collection of user mappings in a database" + + cls = UserMapping + + def from_map(self, server, inusermaps): + """Initialize the dictionary of mappings by examining the input map + + :param server: foreign server associated with mappings + :param inusermaps: input YAML map defining the user mappings + """ + for key in inusermaps: + inobj = inusermaps[key] + self[(server.wrapper, server.name, key)] = UserMapping.from_map( + key, server, inobj) + + def to_map(self, db): + """Convert the user mapping dictionary to a regular dictionary + + :return: dictionary + + Invokes the `to_map` method of each mapping to construct a + dictionary of user mappings. + """ + usermaps = {} + for um in self: + usermaps.update(self[um].to_map(db)) + return usermaps + + +class ForeignTable(Table, DbObjectWithOptions): + """A foreign table definition""" + + privobjtype = "TABLE" + catalog = 'pg_foreign_table' + + def __init__(self, name, schema, description, owner, privileges, + server=None, options={}, + oid=None): + """Initialize the foreign table + + :param name-privileges: see DbClass.__init__ params + :param server: foreign server (from ftserver) + :param options: table options (from ftoptions) + """ + super(ForeignTable, self).__init__(name, schema, description, owner, + privileges) + self.server = server + self.options = {} if options is None else options + self.columns = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, srvname AS server, + ftoptions AS options, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_foreign_table f ON (ftrelid = c.oid) + JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_foreign_server s ON (ftserver = s.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'f' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a foreign table instance from a YAML map + + :param name: foreign table name + :param name: schema name + :param inobj: YAML map of the table + :return: foreign table instance + """ + obj = ForeignTable( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('server', None), inobj.pop('options', {})) + obj.fix_privileges() + return obj + + @property + def objtype(self): + return "FOREIGN TABLE" + + def to_map(self, db, opts): + """Convert a foreign table to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return {} + if len(self.columns) == 0: + return {} + cols = [] + for i in range(len(self.columns)): + col = self.columns[i].to_map(db, opts.no_privs) + if col: + cols.append(col) + tbl = {'columns': cols, 'server': self.server} + if self.description is not None: + tbl.update(description=self.description) + if not opts.no_owner and self.owner is not None: + tbl.update(owner=self.owner) + if len(self.options) > 0: + tbl.update(options=self.options) + if not opts.no_privs: + tbl.update({'privileges': self.map_privs()}) + + return tbl + + @grantable + def create(self, dbversion=None): + """Return SQL statements to CREATE the foreign table + + :return: SQL statements + """ + stmts = [] + cols = [] + options = [] + for col in self.columns: + cols.append(" " + col.add()[0]) + if len(self.options) > 0: + options.append(self.options_clause()) + stmts.append("CREATE FOREIGN TABLE %s (\n%s)\n SERVER %s%s" % ( + self.qualname(), ",\n".join(cols), self.server, + options and '\n ' + ',\n '.join(options) or '')) + if self.owner is not None: + stmts.append(self.alter_owner()) + if self.description is not None: + stmts.append(self.comment()) + for col in self.columns: + if col.description is not None: + stmts.append(col.comment()) + return stmts + + def drop(self): + """Return a SQL DROP statement for the foreign table + + :return: SQL statement + """ + return ["DROP %s %s" % (self.objtype, self.identifier())] + + def alter(self, intable): + """Generate SQL to transform an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + """ + stmts = super(ForeignTable, self).alter(intable) + if intable.owner is not None: + if intable.owner != self.owner: + stmts.append(self.alter_owner(intable.owner)) + stmts.append(self.diff_description(intable)) + return stmts + + +class ForeignTableDict(ClassDict): + "The collection of foreign tables in a database" + + cls = ForeignTable + + def _from_catalog(self): + """Initialize the dictionary of tables by querying the catalogs""" + for tbl in self.fetch(): + self[tbl.key()] = tbl + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + for key in inobjs: + if not key.startswith('foreign table '): + raise KeyError("Unrecognized object type: %s" % key) + ftb = key[14:] + inobj = inobjs[key] + self[(schema.name, ftb)] = ftable = ForeignTable.from_map( + ftb, schema, inobj) + try: + newdb.columns.from_map(ftable, inobj['columns']) + except KeyError as exc: + exc.args = ("Foreign table '%s' has no columns" % ftb, ) + raise + + def link_refs(self, dbcolumns): + """Connect columns to their respective foreign tables + + :param dbcolumns: dictionary of columns + """ + for (sch, tbl) in dbcolumns: + if (sch, tbl) in self: + assert isinstance(self[(sch, tbl)], ForeignTable) + self[(sch, tbl)].columns = dbcolumns[(sch, tbl)] + for col in dbcolumns[(sch, tbl)]: + col._table = self[(sch, tbl)] diff --git a/pyrseas/dbobject/function.py b/pyrseas/dbobject/function.py new file mode 100644 index 0000000..2356261 --- /dev/null +++ b/pyrseas/dbobject/function.py @@ -0,0 +1,732 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.function + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines four classes: Proc derived from + DbSchemaObject, Function and Aggregate derived from Proc, and + FunctionDict derived from DbObjectDict. +""" +from pyrseas.yamlutil import MultiLineStr +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, grantable, split_schema_obj + +VOLATILITY_TYPES = {'i': 'immutable', 's': 'stable', 'v': 'volatile'} +PARALLEL_SAFETY = {'r': 'restricted', 's': 'safe', 'u': 'unsafe'} + + +def split_schema_func(schema, func): + """Split a function related to an object from its schema + + :param schema: schema to which the main object belongs + :param func: possibly qualified function name + :returns: a schema, function tuple, or just the unqualified function name + """ + (sch, fnc) = split_schema_obj(func, schema) + if sch != schema: + return (sch, fnc) + else: + return fnc + + +def join_schema_func(func): + """Join the schema and function, if needed, to form a qualified name + + :param func: a schema, function tuple, or just an unqualified function name + :returns: a possibly-qualified schema.function string + """ + if isinstance(func, tuple): + return "%s.%s" % func + else: + return func + + +class Proc(DbSchemaObject): + """A procedure such as a FUNCTION or an AGGREGATE""" + + keylist = ['schema', 'name', 'arguments'] + catalog = 'pg_proc' + + @property + def allprivs(self): + return 'X' + + def __init__(self, name, schema, description, owner, privileges, + arguments): + """Initialize the procedure + + :param name: function name (from proname) + :param schema: schema name (from pronamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via proowner) + :param privileges: access privileges (from proacl) + :param arguments: argument list (without default values, from + pg_function_identity_arguments) + """ + super(Proc, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + self.arguments = arguments + + def extern_key(self): + """Return the key to be used in external maps for this function + + :return: string + """ + return '%s %s(%s)' % (self.objtype.lower(), self.name, self.arguments) + + def identifier(self): + """Return a full identifier for a function object + + :return: string + """ + return "%s(%s)" % (self.qualname(), self.arguments) + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Proc, self).get_implied_deps(db) + + # Add back the language + if isinstance(self, Function) and getattr(self, 'language', None): + lang = db.languages.get(self.language) + if lang: + deps.add(lang) + + # Add back the types + if self.arguments: + for arg in self.arguments.split(', '): + arg = db.find_type(arg.split()[-1]) + if arg is not None: + deps.add(arg) + + return deps + + +class Function(Proc): + """A procedural language function""" + + def __init__(self, name, schema, description, owner, privileges, + arguments, language, returns, source, obj_file=None, + configuration=None, volatility=None, leakproof=False, + strict=False, security_definer=False, cost=0, rows=0, + allargs=None, oid=None): + """Initialize the function + + :param name-arguments: see Proc.__init__ params + :param language: implementation language (from prolang) + :param returns: return type (from pg_get_function_result/prorettype) + :param source: source code, link symbol, etc. (from prosrc) + :param obj_file: language-specific info (from probin) + :param configuration: configuration variables (from proconfig) + :param volatility: volatility type (from provolatile) + :param leakproof: has side effects (from proleakproof) + :param strict: null handling (from proisstrict) + :param security_definer: security definer (from prosecdef) + :param cost: execution cost estimate (from procost) + :param rows: result row estimate (from prorows) + :param allargs: argument list with defaults (from + pg_get_function_arguments) + """ + super(Function, self).__init__( + name, schema, description, owner, privileges, arguments) + self.language = language + self.returns = returns + if source and '\n' in source: + newsrc = [] + for line in source.split('\n'): + if line and line[-1] in (' ', '\t'): + line = line.rstrip() + newsrc.append(line) + source = '\n'.join(newsrc) + self.source = MultiLineStr(source) + self.obj_file = obj_file + self.configuration = configuration + self.allargs = allargs + if volatility is not None: + self.volatility = volatility[:1].lower() + else: + self.volatility = 'v' + assert self.volatility in VOLATILITY_TYPES.keys() + self.leakproof = leakproof + self.strict = strict + self.security_definer = security_definer + self.cost = cost + self.rows = rows + self.oid = oid + + @staticmethod + def query(dbversion=None): + query = """ + SELECT nspname AS schema, proname AS name, + pg_get_function_identity_arguments(p.oid) AS arguments, + pg_get_function_arguments(p.oid) AS allargs, + pg_get_function_result(p.oid) AS returns, rolname AS owner, + array_to_string(proacl, ',') AS privileges, + l.lanname AS language, provolatile AS volatility, + proisstrict AS strict, prosrc AS source, + probin::text AS obj_file, proconfig AS configuration, + prosecdef AS security_definer, procost AS cost, + proleakproof AS leakproof, prorows::integer AS rows, + obj_description(p.oid, 'pg_proc') AS description, p.oid + FROM pg_proc p JOIN pg_roles r ON (r.oid = proowner) + JOIN pg_namespace n ON (pronamespace = n.oid) + JOIN pg_language l ON (prolang = l.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND %s + AND p.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY nspname, proname""" + if dbversion < 110000: + query = query % "NOT proisagg" + else: + query = query % "prokind = 'f'" + return query + + @staticmethod + def from_map(name, schema, arguments, inobj): + """Initialize a function instance from a YAML map + + :param name: function name + :param name: schema name + :param arguments: arguments + :param inobj: YAML map of the function + :return: function instance + """ + src = inobj.get('source', None) + objfile = inobj.get('obj_file', None) + if (src and objfile) or not (src or objfile): + raise ValueError("Function '%s': either source or obj_file must " + "be specified" % name) + obj = Function( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + arguments, inobj.pop('language', None), + inobj.pop('returns', None), inobj.pop('source', None), + inobj.pop('obj_file', None), + inobj.pop('configuration', None), + inobj.pop('volatility', None), + inobj.pop('leakproof', False), inobj.pop('strict', False), + inobj.pop('security_definer', False), + inobj.pop('cost', 0), inobj.pop('rows', 0), + inobj.pop('allargs', None)) + obj.fix_privileges() + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a function to a YAML-suitable format + + :param no_owner: exclude function owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(Function, self).to_map(db, no_owner, no_privs) + for attr in ('leakproof', 'strict', 'security_definer'): + if dct[attr] is False: + dct.pop(attr) + if self.allargs is None or len(self.allargs) == 0 or \ + self.allargs == self.arguments: + dct.pop('allargs') + if self.configuration is None: + dct.pop('configuration') + if self.volatility == 'v': + dct.pop('volatility') + else: + dct['volatility'] = VOLATILITY_TYPES[self.volatility] + if self.obj_file is not None: + dct['link_symbol'] = self.source + del dct['source'] + else: + del dct['obj_file'] + if self.cost != 0: + if self.language in ['c', 'internal']: + if self.cost == 1: + del dct['cost'] + else: + if self.cost == 100: + del dct['cost'] + else: + del dct['cost'] + if self.rows != 0: + if self.rows == 1000: + del dct['rows'] + else: + del dct['rows'] + + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newsrc=None, basetype=False, returns=None): + """Return SQL statements to CREATE or REPLACE the function + + :param newsrc: new source for a changed function + :return: SQL statements + """ + stmts = [] + if self.obj_file is not None: + src = "'%s', '%s'" % (self.obj_file, + hasattr(self, 'link_symbol') and + self.link_symbol or self.name) + elif self.language == 'internal': + src = "$$%s$$" % (newsrc or self.source) + else: + src = "$_$%s$_$" % (newsrc or self.source) + volat = leakproof = strict = secdef = cost = rows = config = '' + if self.volatility != 'v': + volat = ' ' + VOLATILITY_TYPES[self.volatility].upper() + if self.leakproof is True: + leakproof = ' LEAKPROOF' + if self.strict: + strict = ' STRICT' + if self.security_definer: + secdef = ' SECURITY DEFINER' + if self.configuration is not None: + config = ' SET %s' % self.configuration[0] + if self.cost != 0: + if self.language in ['c', 'internal']: + if self.cost != 1: + cost = " COST %s" % self.cost + else: + if self.cost != 100: + cost = " COST %s" % self.cost + if self.rows != 0: + if self.rows != 1000: + rows = " ROWS %s" % self.rows + + # We may have to create a shell type if we are its input or output + # functions + t = getattr(self, '_defining', None) + if t is not None: + if not hasattr(t, '_shell_created'): + t._shell_created = True + stmts.append("CREATE TYPE %s" % t.qualname()) + + if self.allargs is not None: + args = self.allargs + elif self.arguments is not None: + args = self.arguments + else: + args = '' + stmts.append("CREATE%s FUNCTION %s(%s) RETURNS %s\n LANGUAGE %s" + "%s%s%s%s%s%s%s\n AS %s" % ( + newsrc and " OR REPLACE" or '', self.qualname(), + args, returns or self.returns, self.language, volat, leakproof, + strict, secdef, cost, rows, config, src)) + return stmts + + def alter(self, infunction, dbversion=None, no_owner=False): + """Generate SQL to transform an existing function + + :param infunction: a YAML map defining the new function + :return: list of SQL statements + + Compares the function to an input function and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if self.source != infunction.source and infunction.source is not None: + stmts.append(self.create( + dbversion=dbversion, + returns=infunction.returns, + newsrc=infunction.source, + )) + if self.leakproof is True: + if infunction.leakproof is True: + stmts.append("ALTER FUNCTION %s LEAKPROOF" % self.identifier()) + else: + stmts.append("ALTER FUNCTION %s NOT LEAKPROOF" + % self.identifier()) + elif infunction.leakproof is True: + stmts.append("ALTER FUNCTION %s LEAKPROOF" % self.qualname()) + stmts.append(super(Function, self).alter(infunction, + no_owner=no_owner)) + return stmts + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Function, self).get_implied_deps(db) + + # Add back the return type + rettype = self.returns + if rettype.upper().startswith("SETOF "): + rettype = rettype.split(None, 1)[-1] + rettype = db.find_type(rettype) + if rettype is not None: + deps.add(rettype) + + return deps + + def get_deps(self, db): + deps = super(Function, self).get_deps(db) + + # avoid circular import dependencies + from .dbtype import DbType + + # drop the dependency on the type if this function is an in/out + # because there is a loop here. + for dep in list(deps): + if isinstance(dep, DbType): + for attr in ('input', 'output', 'send', 'receive'): + fname = getattr(dep, attr, None) + if isinstance(fname, tuple): + fname = "%s.%s" % fname + else: + fname = "%s.%s" % (self.schema, fname) + if fname and fname == self.qualname(): + deps.remove(dep) + self._defining = dep # we may need a shell for this + break + + return deps + + def drop(self): + """Generate SQL to drop the current function + + :return: list of SQL statements + """ + # If the function defines a type it will be dropped by the CASCADE + # on the type. + if getattr(self, '_defining', None): + return [] + else: + return super(Function, self).drop() + + +AGGREGATE_KINDS = {'n': 'normal', 'o': 'ordered', 'h': 'hypothetical'} + + +class Aggregate(Proc): + """An aggregate function""" + + def __init__(self, name, schema, description, owner, privileges, + arguments, sfunc, stype, sspace=0, finalfunc=None, + finalfunc_extra=False, initcond=None, sortop=None, + msfunc=None, minvfunc=None, mstype=None, msspace=0, + mfinalfunc=None, mfinalfunc_extra=False, minitcond=None, + kind='normal', combinefunc=None, serialfunc=None, + deserialfunc=None, parallel='unsafe', + oid=None): + """Initialize the aggregate + + :param name-arguments: see Proc.__init__ params + :param sfunc: state transition function (from aggtransfn) + :param stype: state datatype (from aggtranstype) + :param sspace: transition state data size (from aggtransspace) + :param finalfunc: final function (from aggfinalfn) + :param finalfunc_extra: extra args? (from aggfinalextra) + :param initcond: initial value (from agginitval) + :param sortop: sort operator (from aggsortop) + :param msfunc: state transition function (from aggmtransfn) + :param minvfunc: inverse transition function (from aggminvtransfn) + :param mstype: state datatype (from aggmtranstype) + :param msspace: transition state data size (from aggmtransspace) + :param mfinalfunc: final function (from aggfinalfn) + :param mfinalfunc_extra: extra args? (from aggmfinalextra) + :param minitcond: initial value (from aggminitval) + :param kind: aggregate kind (from aggkind) + :param combinefunc: combine function (from aggcombinefn) + :param serialfunc: serialization function (from aggserialfn) + :param deserialfunc: deserialization function (from aggdeserialfn) + :param parallel: parallel safety indicator (from proparallel) + """ + super(Aggregate, self).__init__( + name, schema, description, owner, privileges, arguments) + self.sfunc = split_schema_obj(sfunc, self.schema) + self.stype = self.unqualify(stype) + self.sspace = sspace + if finalfunc is not None and finalfunc != '-': + self.finalfunc = split_schema_obj(finalfunc, self.schema) + else: + self.finalfunc = None + self.finalfunc_extra = finalfunc_extra + self.initcond = initcond + self.sortop = sortop if sortop != '0' else None + if msfunc is not None and msfunc != '-': + self.msfunc = split_schema_obj(msfunc, self.schema) + else: + self.msfunc = None + if minvfunc is not None and minvfunc != '-': + self.minvfunc = split_schema_obj(minvfunc, self.schema) + else: + self.minvfunc = None + if mstype is not None and mstype != '-': + self.mstype = self.unqualify(mstype) + else: + self.mstype = None + self.msspace = msspace + if mfinalfunc is not None and mfinalfunc != '-': + self.mfinalfunc = split_schema_obj(mfinalfunc, self.schema) + else: + self.mfinalfunc = None + self.mfinalfunc_extra = mfinalfunc_extra + self.minitcond = minitcond + if kind is None: + self.kind = 'normal' + elif len(kind) == 1: + self.kind = AGGREGATE_KINDS[kind] + else: + self.kind = kind + assert self.kind in AGGREGATE_KINDS.values() + self.combinefunc = combinefunc if combinefunc != '-' else None + self.serialfunc = serialfunc if serialfunc != '-' else None + self.deserialfunc = deserialfunc if deserialfunc != '-' else None + if parallel is None: + self.parallel = 'unsafe' + elif len(parallel) == 1: + self.parallel = PARALLEL_SAFETY[parallel] + else: + self.parallel = parallel + assert self.parallel in PARALLEL_SAFETY.values() + self.oid = oid + + @staticmethod + def query(dbversion): + query = """ + SELECT nspname AS schema, proname AS name, + pg_get_function_identity_arguments(p.oid) AS arguments, + rolname AS owner, + array_to_string(proacl, ',') AS privileges, + aggtransfn::regproc AS sfunc, + aggtranstype::regtype AS stype, aggtransspace AS sspace, + aggfinalfn::regproc AS finalfunc, + aggfinalextra AS finalfunc_extra, + agginitval AS initcond, aggsortop::regoper AS sortop, + aggmtransfn::regproc AS msfunc, + aggminvtransfn::regproc AS minvfunc, + aggmtranstype::regtype AS mstype, + aggmtransspace AS msspace, + aggmfinalfn::regproc AS mfinalfunc, + aggmfinalextra AS mfinalfunc_extra, + aggminitval AS minitcond, aggkind AS kind, + aggcombinefn AS combinefunc, + aggserialfn AS serialfunc, aggdeserialfn AS deserialfunc, + proparallel AS parallel,%s + obj_description(p.oid, 'pg_proc') AS description, p.oid + FROM pg_proc p JOIN pg_roles r ON (r.oid = proowner) + JOIN pg_namespace n ON (pronamespace = n.oid) + LEFT JOIN pg_aggregate a ON (p.oid = aggfnoid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + %s + AND p.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY nspname, proname""" + extra = ("", "") + if dbversion < 110000: + extra = (" proisagg,", "") + else: + extra = ("", "AND prokind = 'a'") + return query % extra + + @staticmethod + def from_map(name, schema, arguments, inobj): + """Initialize an aggregate instance from a YAML map + + :param name: aggregate name + :param name: schema name + :param arguments: arguments + :param inobj: YAML map of the aggregate + :return: aggregate instance + """ + obj = Aggregate( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + arguments, inobj.get('sfunc'), inobj.get('stype'), + inobj.pop('sspace', 0), inobj.pop('finalfunc', None), + inobj.pop('finalfunc_extra', False), inobj.pop('initcond', None), + inobj.pop('sortop', None), inobj.pop('msfunc', None), + inobj.pop('minvfunc', None), inobj.pop('mstype', None), + inobj.pop('msspace', 0), inobj.pop('mfinalfunc', None), + inobj.pop('mfinalfunc_extra', False), + inobj.pop('minitcond', None), inobj.pop('kind', 'normal'), + inobj.pop('combinefunc', None), inobj.pop('serialfunc', None), + inobj.pop('deseriafunc', None), inobj.pop('parallel', 'unsafe')) + obj.fix_privileges() + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert an aggregate to a YAML-suitable format + + :param no_owner: exclude aggregate owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(Aggregate, self).to_map(db, no_owner, no_privs) + dct['sfunc'] = self.unqualify(join_schema_func(self.sfunc)) + for attr in ('finalfunc', 'msfunc', 'minvfunc', 'mfinalfunc'): + if getattr(self, attr) is None: + dct.pop(attr) + else: + dct[attr] = self.unqualify( + join_schema_func(getattr(self, attr))) + for attr in ('initcond', 'sortop', 'minitcond', 'mstype', + 'combinefunc', 'serialfunc', 'deserialfunc'): + if getattr(self, attr) is None: + dct.pop(attr) + for attr in ('sspace', 'msspace'): + if getattr(self, attr) == 0: + dct.pop(attr) + for attr in ('finalfunc_extra', 'mfinalfunc_extra'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.kind == 'normal': + dct.pop('kind') + if self.parallel == 'unsafe': + dct.pop('parallel') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the aggregate + + :param dbversion: Posgres version + :return: SQL statements + """ + opt_clauses = [] + if self.finalfunc is not None: + opt_clauses.append("FINALFUNC = %s" % + join_schema_func(self.finalfunc)) + if self.initcond is not None: + opt_clauses.append("INITCOND = '%s'" % self.initcond) + if self.combinefunc is not None: + opt_clauses.append("COMBINEFUNC = %s" % self.combinefunc) + if self.serialfunc is not None: + opt_clauses.append("SERIALFUNC = %s" % self.serialfunc) + if self.deserialfunc is not None: + opt_clauses.append("DESERIALFUNC = %s" % self.deserialfunc) + if self.sspace > 0: + opt_clauses.append("SSPACE = %d" % self.sspace) + if self.finalfunc_extra: + opt_clauses.append("FINALFUNC_EXTRA") + if self.msfunc is not None: + opt_clauses.append("MSFUNC = %s" % join_schema_func(self.msfunc)) + if self.minvfunc is not None: + opt_clauses.append("MINVFUNC = %s" % join_schema_func(self.minvfunc)) + if self.mstype is not None: + opt_clauses.append("MSTYPE = %s" % self.mstype) + if self.msspace > 0: + opt_clauses.append("MSSPACE = %d" % self.msspace) + if self.mfinalfunc is not None: + opt_clauses.append("MFINALFUNC = %s" % + join_schema_func(self.mfinalfunc)) + if self.mfinalfunc_extra: + opt_clauses.append("MFINALFUNC_EXTRA") + if self.minitcond is not None: + opt_clauses.append("MINITCOND = '%s'" % self.minitcond) + if self.kind == 'hypothetical': + opt_clauses.append("HYPOTHETICAL") + if self.sortop is not None: + clause = self.sortop + if not clause.startswith('OPERATOR'): + clause = "OPERATOR(%s)" % clause + opt_clauses.append("SORTOP = %s" % clause) + if self.parallel != 'unsafe': + opt_clauses.append("PARALLEL = %s" % self.parallel.upper()) + return ["CREATE AGGREGATE %s(%s) (\n SFUNC = %s," + "\n STYPE = %s%s%s)" % ( + self.qualname(), self.arguments, + join_schema_func(self.sfunc), self.stype, + opt_clauses and ',\n ' or '', + ',\n '.join(opt_clauses))] + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Aggregate, self).get_implied_deps(db) + + if isinstance(self.sfunc, tuple): + sch, fnc = self.sfunc + else: + sch, fnc = self.schema, self.sfunc + if 'ORDER BY' in self.arguments: + args = self.arguments.replace(' ORDER BY', ',') + else: + args = self.stype + ', ' + self.arguments + deps.add(db.functions[sch, fnc, args]) + for fn in ('finalfunc', 'mfinalfunc'): + if getattr(self, fn) is not None: + func = getattr(self, fn) + if isinstance(func, tuple): + sch, fnc = func + else: + sch, fnc = self.schema, func + deps.add(db.functions[sch, fnc, self.mstype + if fn[0] == 'm' else self.stype]) + for fn in ('msfunc', 'minvfunc'): + if getattr(self, fn) is not None: + func = getattr(self, fn) + if isinstance(func, tuple): + sch, fnc = func + else: + sch, fnc = self.schema, func + args = self.mstype + ", " + self.arguments + deps.add(db.functions[sch, fnc, args]) + + return deps + + +class ProcDict(DbObjectDict): + "The collection of regular and aggregate functions in a database" + + cls = Proc + + def _from_catalog(self): + """Initialize the dictionary of procedures by querying the catalogs""" + for cls in (Function, Aggregate): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, infuncs): + """Initialize the dictionary of functions by converting the input map + + :param schema: schema owning the functions + :param infuncs: YAML map defining the functions + """ + for key in infuncs: + (objtype, spc, fnc) = key.partition(' ') + if spc != ' ' or objtype not in ['function', 'aggregate']: + raise KeyError("Unrecognized object type: %s" % key) + paren = fnc.find('(') + if paren == -1 or fnc[-1:] != ')': + raise KeyError("Invalid function signature: %s" % fnc) + arguments = fnc[paren + 1:-1] + inobj = infuncs[key] + fnc = fnc[:paren] + if objtype == 'function': + func = Function.from_map(fnc, schema, arguments, inobj) + else: + func = Aggregate.from_map(fnc, schema, arguments, inobj) + self[(schema.name, fnc, arguments)] = func + + def find(self, func, args): + """Return a function given its name and arguments + + :param func: name of the function, eventually with schema + :param args: list of type names + + Return the function found, else None. + """ + schema, name = split_schema_obj(func) + args = ', '.join(args) + return self.get((schema, name, args)) + + def link_refs(self, dbtypes): + """Connect the functions to other objects + + - Connect defining functions to the type they define + + :param dbtypes: dictionary of types + """ + # TODO: this link is needed from map, not from sql. + # is this a pattern? I was assuming link_refs would have disappeared + # but I'm actually still maintaining them. Verify if they are always + # only used for from_map, not for from_catalog + for key in dbtypes: + t = dbtypes[key] + for f in t.find_defining_funcs(self): + f._defining = t diff --git a/pyrseas/dbobject/index.py b/pyrseas/dbobject/index.py new file mode 100644 index 0000000..c87c938 --- /dev/null +++ b/pyrseas/dbobject/index.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.index + ~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Index and IndexDict, derived + from DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable + + +def split_exprs(idx_exprs): + "Helper function to split index expressions from pg_get_expr()" + level = 0 + in_literal = False + splits = [] + + # Split around commas but only at the first parens level. + # TODO: you can still fool this function with a string containing a quote. + start = 0 + for i, c in enumerate(idx_exprs): + if c == "'": + in_literal = not in_literal + continue + if in_literal: + continue + elif c == '(': + level += 1 + elif c == ')': + level -= 1 + elif c == ',' and level == 0: + splits.append((start, i)) + start = i + 1 + for s in idx_exprs[start:]: + if s == ' ': + start += 1 + else: + break + splits.append((start, i+1)) + return [idx_exprs[start:end] for start, end in splits] + + +class Index(DbSchemaObject): + """A physical index definition, other than a primary key or unique + constraint index. + + An index is identified by its schema name and index name. However, + at this time, Pyrseas uses the triple schema-table-index names as the + identifier. + """ + # TODO: This should be fixed in this or a subsequent release. + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_index' + + def __init__(self, name, schema, table, description, unique=False, + access_method='btree', keys=[], predicate=None, + tablespace=None, cluster=False, keyexprs=None, defn=None, + oid=None): + """Initialize the index + + :param name: index name (from relname) + :param schema: schema name (from nspname via relnamespace) + :param table: table name (from indrelid) + :param description: comment text (from obj_description) + :param unique: unique indicator (from indisunique) + :param access_method: access method (from amname via relam) + :param keys: list of columns (from indkey) + :param predicate: partial index predicate (from indpred) + :param tablespace: tablespace name (from spcname via reltablespace) + :param cluster: clustered indicator (from indisclustered) + :param keyexprs: list of expressions (from indexprs) + :param defn: index definition (from pg_get_indexdef) + """ + super(Index, self).__init__(name, schema, description) + self.table = self.unqualify(table) + self.unique = unique + self.access_method = access_method + if defn is not None: + self.keys = self._parse_keys(keys, keyexprs, defn) + else: + self.keys = keys + self.predicate = predicate + self.tablespace = tablespace + self.cluster = cluster + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, indrelid::regclass AS table, + c.relname AS name, amname AS access_method, + indisunique AS unique, indkey AS keys, + pg_get_expr(indexprs, indrelid) AS keyexprs, + pg_get_expr(indpred, indrelid) AS predicate, + pg_get_indexdef(indexrelid) AS defn, + spcname AS tablespace, indisclustered AS cluster, + obj_description (c.oid, 'pg_class') AS description, c.oid + FROM pg_index i JOIN pg_class c ON (indexrelid = c.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + JOIN pg_am ON (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (c.reltablespace = t.oid) + WHERE NOT indisprimary AND c.relpersistence != 't' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE contype in ('p', 'u') + AND conindid = c.oid) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize an index instance from a YAML map + + :param name: index name + :param table: map of table + :param inobj: YAML map of the index + :return: Index instance + """ + keys = 'keys' + if 'columns' in inobj: + keys = 'columns' + elif 'keys' not in inobj: + raise KeyError("Index '%s' is missing keys specification" % name) + obj = Index( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('unique', False), inobj.pop('access_method', 'btree'), + inobj.pop(keys, []), inobj.pop('predicate', None), + inobj.pop('tablespace', None), inobj.pop('cluster', False)) + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + obj.set_oldname(inobj) + return obj + + def _parse_keys(self, keycols, exprs, defn): + keydefs, _, _ = defn.partition(' WHERE ') + _, _, keydefs = keydefs.partition(' USING ') + keydefs = keydefs[keydefs.find(' (') + 2:-1] + # split expressions + if exprs is not None: + exprs = split_exprs(exprs) + i = 0 + rest = keydefs + keys = [] + for col in keycols.split(): + keyopts = [] + extra = {} + if col == '0': + expr = exprs[i] + if rest and rest[0] == '(': + expr = '(' + expr + ')' + assert(rest.startswith(expr)) + key = expr + extra = {'type': 'expression'} + explen = len(expr) + loc = rest[explen:].find(',') + if loc == 0: + keyopts = [] + rest = rest[explen + 1:].lstrip() + elif loc == -1: + keyopts = rest[explen:].split() + rest = '' + else: + keyopts = rest[explen:explen + loc].split() + rest = rest[explen + loc + 1:].lstrip() + i += 1 + else: + loc = rest.find(',') + key = rest[:loc] if loc != -1 else rest.lstrip() + keyopts = key.split()[1:] + key = key.split()[0] + rest = rest[loc + 1:] + rest = rest.lstrip() + skipnext = False + for j, opt in enumerate(keyopts): + if skipnext: + skipnext = False + continue + if opt.upper() not in ['COLLATE', 'ASC', 'DESC', 'NULLS', + 'FIRST', 'LAST']: + extra.update(opclass=opt) + continue + elif opt == 'COLLATE': + extra.update(collation=keyopts[j + 1]) + skipnext = True + elif opt == 'NULLS': + extra.update(nulls=keyopts[j + 1].lower()) + skipnext = True + elif opt == 'DESC': + extra.update(order='desc') + if extra: + key = {key: extra} + keys.append(key) + return keys + + def key_expressions(self): + """Return comma-separated list of key column names and qualifiers + + :return: string + """ + colspec = [] + for col in self.keys: + if isinstance(col, str): + colspec.append(col) + else: + clause = list(col.keys())[0] + vals = list(col.values())[0] + if 'collation' in vals: + clause += ' COLLATE ' + vals['collation'] + if 'opclass' in vals: + clause += ' ' + vals['opclass'] + if 'order' in vals: + clause += ' ' + vals['order'].upper() + if 'nulls' in vals: + clause += ' NULLS ' + vals['nulls'].upper() + colspec.append(clause) + return ", ".join(colspec) + + def to_map(self, db): + """Convert an index definition to a YAML-suitable format + + :return: dictionary + """ + dct = super(Index, self).to_map(db) + if self.access_method == 'btree': + dct.pop('access_method') + if not self.unique: + dct.pop('unique') + for attr in ['predicate', 'tablespace']: + if getattr(self, attr) is None: + dct.pop(attr) + if not self.cluster: + dct.pop('cluster') + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return a SQL statement to CREATE the index + + :return: SQL statements + """ + stmts = [] + + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return stmts + + acc = '' + if self.access_method != 'btree': + acc = 'USING %s ' % self.access_method + tblspc = '' + if self.tablespace is not None: + tblspc = '\n TABLESPACE %s' % self.tablespace + pred = '' + if self.predicate is not None: + pred = '\n WHERE %s' % self.predicate + stmts.append("CREATE %sINDEX %s ON %s %s(%s)%s%s" % ( + 'UNIQUE ' if self.unique else '', quote_id(self.name), + self.qualname(self.schema, self.table), acc, + self.key_expressions(), tblspc, pred)) + if self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self.qualname(self.schema, self.table), quote_id(self.name))) + return stmts + + def alter(self, inindex): + """Generate SQL to transform an existing index + + :param inindex: a YAML map defining the new index + :return: list of SQL statements + + Compares the index to an input index and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return stmts + + if self.access_method != inindex.access_method \ + or self.unique != inindex.unique \ + or self.keys != inindex.keys: + stmts.append("DROP INDEX %s" % self.qualname()) + self.access_method = inindex.access_method + self.unique = inindex.unique + self.keys = inindex.keys + stmts.append(self.create()) + + base = "ALTER INDEX %s\n " % self.qualname() + if inindex.tablespace is not None: + if self.tablespace is not None \ + or self.tablespace != inindex.tablespace: + stmts.append(base + "SET TABLESPACE %s" + % quote_id(inindex.tablespace)) + elif self.tablespace is not None: + stmts.append(base + "SET TABLESPACE pg_default") + if inindex.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self.qualname(self.schema, self.table), + quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self.qualname(self.schema, self.table)) + stmts.append(super(Index, self).alter(inindex)) + return stmts + + def drop(self): + """Generate SQL to drop the current index + + :return: list of SQL statements + """ + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return [] + + return ["DROP INDEX %s" % self.identifier()] + + def get_implied_deps(self, db): + deps = super(Index, self).get_implied_deps(db) + + # add the table we are defined into + deps.add(db.tables[self.schema, self.table]) + # TODO: add column collation specs if present + + return deps + + +class IndexDict(DbObjectDict): + "The collection of indexes on tables in a database" + + cls = Index + + def _from_catalog(self): + """Initialize the dictionary of indexes by querying the catalogs""" + self.query = self.cls.query() + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, table, inindexes): + """Initialize the dictionary of indexes by converting the input map + + :param table: table owning the indexes + :param inindexes: YAML map defining the indexes + """ + for i in inindexes: + inobj = inindexes[i] + self[(table.schema, table.name, i)] = Index.from_map( + i, table, inobj) diff --git a/pyrseas/dbobject/language.py b/pyrseas/dbobject/language.py new file mode 100644 index 0000000..9f17b10 --- /dev/null +++ b/pyrseas/dbobject/language.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.language + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Language and LanguageDict, derived from + DbObject and DbObjectDict, respectively. + + See note at + https://www.postgresql.org/docs/current/static/sql-createlanguage.html + regarding status of procedural languages since Postgres 9.1. +""" +from . import DbObjectDict, DbObject, quote_id +from .function import Function +from .extension import CORE_LANGS + + +class Language(DbObject): + """A procedural language definition""" + + keylist = ['name'] + single_extern_file = True + catalog = 'pg_language' + + def __init__(self, name, description=None, owner=None, privileges=[], + trusted=False, + oid=None): + """Initialize the language + + :param name: language name (from lanname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via lanowner) + :param privileges: access privileges (from lanacl) + :param trusted: is this a trusted language? (from lanpltrusted) + """ + super(Language, self).__init__(name, description) + self._init_own_privs(owner, privileges) + self.trusted = trusted + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT lanname AS name, lanpltrusted AS trusted, rolname AS owner, + array_to_string(lanacl, ',') AS privileges, + obj_description(l.oid, 'pg_language') AS description, l.oid + FROM pg_language l JOIN pg_roles r ON (r.oid = lanowner) + WHERE lanispl + ORDER BY lanname""" + + @staticmethod + def from_map(name, inobj): + """Initialize a Language instance from a YAML map + + :param name: Language name + :param inobj: YAML map of the Language + :return: Language instance + """ + obj = Language( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('privileges', []), inobj.pop('trusted', False)) + obj.fix_privileges() + if '_ext' in inobj: + obj._ext = inobj['_ext'] + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert language to a YAML-suitable format + + :param no_owner: exclude language owner information + :return: dictionary + """ + if hasattr(self, '_ext'): + return None + if self.name in CORE_LANGS: + return None + dct = super(Language, self).to_map(db, no_owner, no_privs) + if 'functions' in dct: + del dct['functions'] + return dct + + def create(self, dbversion=None): + """Return SQL statements to CREATE the language + + :return: empty, because PL's should all be extensions + """ + return [] + + def alter(self, inobj): + if self.name in CORE_LANGS: + return [] + return super(Language, self).alter(inobj) + + def drop(self): + if self.name in CORE_LANGS: + return [] + return super(Language, self).drop() + + +class LanguageDict(DbObjectDict): + "The collection of procedural languages in a database." + + cls = Language + + def from_map(self, inmap): + """Initialize the dictionary of languages by examining the input map + + :param inmap: the input YAML map defining the languages + """ + for key in inmap: + (objtype, spc, lng) = key.partition(' ') + if spc != ' ' or objtype != 'language': + raise KeyError("Unrecognized object type: %s" % key) + inobj = inmap[key] + self[lng] = Language.from_map(lng, inobj) + + def link_refs(self, dbfunctions, langs): + """Connect functions to their respective languages + + :param dbfunctions: dictionary of functions + + Fills in the `functions` dictionary for each language by + traversing the `dbfunctions` dictionary, which is keyed by + schema and function name. + """ + for (sch, fnc, arg) in dbfunctions: + func = dbfunctions[(sch, fnc, arg)] + if not isinstance(func, Function) or ( + func.language in ['sql', 'c', 'internal']): + continue + try: + language = self[(func.language)] + except KeyError as exc: + if func.language in langs: + continue + raise exc + if not hasattr(language, 'functions'): + language.functions = {} + language.functions.update({fnc: func}) diff --git a/pyrseas/dbobject/operator.py b/pyrseas/dbobject/operator.py new file mode 100644 index 0000000..b55732d --- /dev/null +++ b/pyrseas/dbobject/operator.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operator + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Operator derived from + DbSchemaObject and OperatorDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, ownable +from . import split_schema_obj, split_func_args + + +class Operator(DbSchemaObject): + """An operator""" + + keylist = ['schema', 'name', 'leftarg', 'rightarg'] + single_extern_file = True + catalog = 'pg_operator' + + def __init__(self, name, schema, description, owner, procedure, + leftarg=None, rightarg=None, commutator=None, negator=None, + restrict=None, join=None, hashes=False, merges=False, + oid=None): + """Initialize the operator + + :param name: operator name (from oprname) + :param description: comment text (from obj_description()) + :param schema: schema name (from oprnamespace) + :param owner: owner name (from rolname via oprowner) + :param procedure: implementor function (from oprcode) + :param leftarg: left operand type (from oprleft) + :param rightarg: right operand type (from oprright) + :param commutator: commutator, if any (from oprcom) + :param negator: negator, if any (from oprnegate) + :param restrict: restriction selectivity function (from oprrest) + :param join: join selectivity function (from oprjoin) + :param hashes: supports hash joins? (from oprcanhash) + :param merges: support merge joins? (from oprcanmerge) + """ + super(Operator, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.procedure = procedure + self.leftarg = leftarg if leftarg != '-' else None + self.rightarg = rightarg if rightarg != '-' else None + self.commutator = commutator if commutator != '0' else None + self.negator = negator if negator != '0' else None + self.restrict = restrict if restrict != '-' else None + self.join = join if join != '-' else None + self.hashes = hashes + self.merges = merges + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, oprname AS name, rolname AS owner, + oprleft::regtype AS leftarg, oprright::regtype AS rightarg, + oprcode AS procedure, oprcom::regoper AS commutator, + oprnegate::regoper AS negator, oprrest AS restrict, + oprjoin AS join, oprcanhash AS hashes, + oprcanmerge AS merges, + obj_description(o.oid, 'pg_operator') AS description, o.oid + FROM pg_operator o JOIN pg_roles r ON (r.oid = oprowner) + JOIN pg_namespace n ON (oprnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_operator'::regclass) + ORDER BY nspname, oprname""" + + @staticmethod + def from_map(name, schema, leftarg, rightarg, inobj): + """Initialize an operator instance from a YAML map + + :param name: operator name + :param name: schema name + :param leftarg: left-hand argument + :param rightarg: right-hand argument + :param inobj: YAML map of the operator + :return: operator instance + """ + obj = Operator( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('procedure', None), + leftarg, rightarg, inobj.pop('commutator', None), + inobj.pop('negator', None), inobj.pop('restrict', None), + inobj.pop('join', None), inobj.pop('hashes', False), + inobj.pop('merges', None)) + obj.set_oldname(inobj) + return obj + + def extern_key(self): + """Return the key to be used in external maps for this operator + + :return: string + """ + return '%s %s(%s, %s)' % ( + self.objtype.lower(), self.name, + 'NONE' if self.leftarg is None else self.leftarg, + 'NONE' if self.rightarg is None else self.rightarg) + + def qualname(self): + """Return the schema-qualified name of the operator + + :return: string + + No qualification is used if the schema is 'public'. + """ + return self.schema == 'public' and self.name \ + or "%s.%s" % (quote_id(self.schema), self.name) + + def identifier(self): + """Return a full identifier for an operator object + + :return: string + """ + return "%s(%s, %s)" % (self.qualname(), self.leftarg, self.rightarg) + + def to_map(self, db, no_owner=False): + """Convert an operator to a YAML-suitable format + + :param db: db used to tie the objects together + :param no_owner: exclude object owner information + :return: dictionary + """ + dct = super(Operator, self).to_map(db, no_owner) + for attr in ['commutator', 'join', 'negator', 'restrict']: + if dct[attr] is None: + dct.pop(attr) + for attr in ['hashes', 'merges']: + if dct[attr] is False: + dct.pop(attr) + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE or REPLACE the operator + + :return: SQL statements + """ + opt_clauses = [] + if self.leftarg is not None: + opt_clauses.append("LEFTARG = %s" % self.leftarg) + if self.rightarg is not None: + opt_clauses.append("RIGHTARG = %s" % self.rightarg) + if self.commutator is not None: + opt_clauses.append("COMMUTATOR = OPERATOR(%s)" % self.commutator) + if self.negator is not None: + opt_clauses.append("NEGATOR = OPERATOR(%s)" % self.negator) + if self.restrict is not None: + opt_clauses.append("RESTRICT = %s" % self.restrict) + if self.join is not None: + opt_clauses.append("JOIN = %s" % self.join) + if self.hashes: + opt_clauses.append("HASHES") + if self.merges: + opt_clauses.append("MERGES") + return ["CREATE OPERATOR %s (\n PROCEDURE = %s%s%s)" % ( + self.qualname(), self.procedure, + ',\n ' if opt_clauses else '', ',\n '.join(opt_clauses))] + + def get_implied_deps(self, db): + deps = super(Operator, self).get_implied_deps(db) + + # Types may be not found because builtin, or the operator unary + if self.leftarg is not None: + leftarg = db.types.find(self.leftarg) + if leftarg: + deps.add(leftarg) + + if self.rightarg is not None: + rightarg = db.types.find(self.rightarg) + if rightarg: + deps.add(rightarg) + + # The function instead we expect it exists + # TODO: another ugly hack to locate the object + fschema, fname = split_schema_obj(self.procedure, self.schema) + fargs = ', '.join(t for t in [self.leftarg, self.rightarg] + if t is not None) + if (fschema, fname, fargs) in db.functions: + func = db.functions[fschema, fname, fargs] + deps.add(func) + + # This helper function may be a builtin + if self.restrict is not None: + fschema, fname = split_schema_obj(self.restrict) + func = db.functions.get((fschema, fname, + "internal, oid, internal, integer")) + if func: + deps.add(func) + + return deps + + +class OperatorDict(DbObjectDict): + "The collection of operators in a database" + + cls = Operator + + def find(self, oper): + """Return an operator given its signature + + :param oper: a signature such as '#>=#(hstore,hstore)' + + Return the operator found, else None. + """ + schema, name = split_schema_obj(oper) + name, args = split_func_args(name) + return self.get((schema, name) + tuple(args)) + + def from_map(self, schema, inopers): + """Initialize the dictionary of operators by converting the input map + + :param schema: schema owning the operators + :param inopers: YAML map defining the operators + """ + for key in inopers: + (objtype, spc, opr) = key.partition(' ') + if spc != ' ' or objtype != 'operator': + raise KeyError("Unrecognized object type: %s" % key) + paren = opr.find('(') + if paren == -1 or opr[-1:] != ')': + raise KeyError("Invalid operator signature: %s" % opr) + (leftarg, rightarg) = opr[paren + 1:-1].split(',') + if leftarg == 'NONE': + leftarg = None + rightarg = rightarg.lstrip() + if rightarg == 'NONE': + rightarg = None + inobj = inopers[key] + opr = opr[:paren] + self[(schema.name, opr, leftarg, rightarg)] = Operator.from_map( + opr, schema, leftarg, rightarg, inobj) diff --git a/pyrseas/dbobject/operclass.py b/pyrseas/dbobject/operclass.py new file mode 100644 index 0000000..5d874b3 --- /dev/null +++ b/pyrseas/dbobject/operclass.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operclass + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: OperatorClass derived from + DbSchemaObject and OperatorClassDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_func_args, split_schema_obj + + +class OperatorClass(DbSchemaObject): + """An operator class""" + + keylist = ['schema', 'name', 'index_method'] + single_extern_file = True + catalog = 'pg_opclass' + + def __init__(self, name, schema, index_method, description, owner, + family, type, default=None, storage=None, oid=None): + """Initialize the operator class + + :param name: operator name (from opcname) + :param schema: schema name (from opcnamespace) + :param index_method: index access method (from amname via opcmethod) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via opcowner) + :param family: operator family (from opfname via opcfamily) + :param type: data type indexed (from opcintype) + :param default: default class for this type? (from opcdefault) + :param storage: type of data stored (from opckeytype) + """ + super(OperatorClass, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.index_method = index_method + self.family = family + (sch, typ) = split_schema_obj(type) + self.type = typ if self.schema == sch else type + self.default = default + self.storage = storage if storage != '-' else None + self.operators = {} + self.functions = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, opcname AS name, rolname AS owner, + amname AS index_method, opfname AS family, + opcintype::regtype AS type, opcdefault AS default, + opckeytype::regtype AS storage, + obj_description(o.oid, 'pg_opclass') AS description, o.oid + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_roles r ON (r.oid = opcowner) + JOIN pg_opfamily f ON (opcfamily = f.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname""" + + @staticmethod + def opquery(): + return """ + SELECT nspname AS schema, opcname AS name, amname AS index_method, + amopstrategy AS strategy, amopopr::regoperator AS operator + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid), pg_amop ao, + pg_depend + WHERE refclassid = 'pg_opclass'::regclass + AND classid = 'pg_amop'::regclass AND objid = ao.oid + AND refobjid = o.oid + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname, amopstrategy""" + + @staticmethod + def prquery(): + return """ + SELECT nspname AS schema, opcname AS name, amname AS index_method, + amprocnum AS support, amproc::regprocedure AS function + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid), pg_amproc ap, + pg_depend + WHERE refclassid = 'pg_opclass'::regclass + AND classid = 'pg_amproc'::regclass AND objid = ap.oid + AND refobjid = o.oid + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname, amprocnum""" + + @staticmethod + def from_map(name, schema, index_method, inobj): + """Initialize an operator class instance from a YAML map + + :param name: operator class name + :param name: schema name + :param index_method: index method + :param inobj: YAML map of the operator class + :return: operator class instance + """ + obj = OperatorClass( + name, schema.name, index_method, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('family', None), + inobj.pop('type', None), inobj.pop('default', False), + inobj.pop('storage', None)) + if 'operators' in inobj: + obj.operators = inobj.get('operators') + if 'functions' in inobj: + obj.functions = inobj.get('functions') + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "OPERATOR CLASS" + + def extern_key(self): + """Return the key to be used in external maps for this operator + + :return: string + """ + return '%s %s using %s' % (self.objtype.lower(), self.name, + self.index_method) + + def identifier(self): + """Return a full identifier for an operator class + + :return: string + """ + return "%s USING %s" % (self.qualname(), self.index_method) + + def to_map(self, db, no_owner): + """Convert operator class to a YAML-suitable format + + :return: dictionary + """ + dct = super(OperatorClass, self).to_map(db, no_owner) + if self.storage is None: + del dct['storage'] + if not self.default: + del dct['default'] + if self.name == self.family: + del dct['family'] + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the operator class + + :return: SQL statements + """ + dflt = '' + if self.default: + dflt = "DEFAULT " + clauses = [] + for (strat, oper) in list(self.operators.items()): + clauses.append("OPERATOR %d %s" % (strat, oper)) + for (supp, func) in list(self.functions.items()): + clauses.append("FUNCTION %d %s" % (supp, func)) + if self.storage is not None: + clauses.append("STORAGE %s" % self.storage) + return ["CREATE OPERATOR CLASS %s\n %sFOR TYPE %s USING %s " + "AS\n %s" % ( + self.qualname(), dflt, + self.qualname(self.schema, self.type), self.index_method, + ',\n ' .join(clauses))] + + def get_implied_deps(self, db): + deps = super(OperatorClass, self).get_implied_deps(db) + + type = db.types.find(self.type) + if type: + deps.add(type) + + if self.storage is not None: + type = db.types.find(self.storage) + if type: + deps.add(type) + + for f in self.functions.values(): + f = db.functions.find(*split_func_args(f)) + if f is not None: + deps.add(f) + + for f in self.operators.values(): + f = db.operators.find(f) + if f is not None: + deps.add(f) + + if self.family is not None: + f = db.operfams.find(self.schema, self.family, self.index_method) + if f is not None: + deps.add(f) + + return deps + + +class OperatorClassDict(DbObjectDict): + "The collection of operator classes in a database" + + cls = OperatorClass + + def _from_catalog(self): + """Initialize the dictionary of operator classes from the catalogs""" + for opclass in self.fetch(): + self[opclass.key()] = opclass + opers = self.dbconn.fetchall(self.cls.opquery()) + self.dbconn.rollback() + for opdata in opers: + sch = opdata["schema"] + opc = opdata["name"] + idx = opdata["index_method"] + strat = opdata["strategy"] + oper = opdata["operator"] + opcls = self[(sch, opc, idx)] + opcls.operators.update({strat: oper}) + funcs = self.dbconn.fetchall(self.cls.prquery()) + self.dbconn.rollback() + for oprdata in funcs: + sch = oprdata["schema"] + opc = oprdata["name"] + idx = oprdata["index_method"] + supp = oprdata["support"] + func = oprdata["function"] + opcls = self[(sch, opc, idx)] + opcls.functions.update({supp: func}) + + def from_map(self, schema, inopcls): + """Initialize the dictionary of operator classes from the input map + + :param schema: schema owning the operator classes + :param inopcls: YAML map defining the operator classes + """ + for key in inopcls: + if not key.startswith('operator class ') or ' using ' not in key: + raise KeyError("Unrecognized object type: %s" % key) + pos = key.rfind(' using ') + opc = key[15:pos] # 15 = len('operator class ') + idx = key[pos + 7:] # 7 = len(' using ') + inobj = inopcls[key] + self[(schema.name, opc, idx)] = OperatorClass.from_map( + opc, schema, idx, inobj) diff --git a/pyrseas/dbobject/operfamily.py b/pyrseas/dbobject/operfamily.py new file mode 100644 index 0000000..e9e3277 --- /dev/null +++ b/pyrseas/dbobject/operfamily.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operfamily + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: OperatorFamily derived from + DbSchemaObject and OperatorFamilyDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_schema_obj + + +class OperatorFamily(DbSchemaObject): + """An operator family""" + + keylist = ['schema', 'name', 'index_method'] + single_extern_file = True + catalog = 'pg_opfamily' + + def __init__(self, name, schema, index_method, description, owner, + oid=None): + """Initialize the operator family + + :param name: operator name (from opfname) + :param schema: schema name (from opfnamespace) + :param index_method: index access method (from amname via opfmethod) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via opfowner) + """ + super(OperatorFamily, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.index_method = index_method + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, opfname AS name, rolname AS owner, + amname AS index_method, + obj_description(o.oid, 'pg_opfamily') AS description, o.oid + FROM pg_opfamily o JOIN pg_roles r ON (r.oid = opfowner) + JOIN pg_am a ON (opfmethod = a.oid) + JOIN pg_namespace n ON (opfnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opfamily'::regclass) + ORDER BY opfnamespace, opfname, amname""" + + @staticmethod + def from_map(name, schema, index_method, inobj): + """Initialize an operator family instance from a YAML map + + :param name: operator family name + :param name: schema name + :param index_method: index method + :param inobj: YAML map of the operator family + :return: operator family instance + """ + obj = OperatorFamily( + name, schema.name, index_method, inobj.pop('description', None), + inobj.pop('owner', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "OPERATOR FAMILY" + + def extern_key(self): + """Return the key to be used in external maps for the operator family + + :return: string + """ + return '%s %s using %s' % (self.objtype.lower(), self.name, + self.index_method) + + def identifier(self): + """Return a full identifier for an operator family object + + :return: string + """ + return "%s USING %s" % (self.qualname(), self.index_method) + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the operator family + + :return: SQL statements + """ + return ["CREATE OPERATOR FAMILY %s USING %s" % ( + self.qualname(), self.index_method)] + + +class OperatorFamilyDict(DbObjectDict): + "The collection of operator families in a database" + + cls = OperatorFamily + + def from_map(self, schema, inopfams): + """Initialize the dict of operator families by converting the input map + + :param schema: schema owning the operators + :param inopfams: YAML map defining the operator families + """ + for key in inopfams: + if not key.startswith('operator family ') or ' using ' not in key: + raise KeyError("Unrecognized object type: %s" % key) + pos = key.rfind(' using ') + opf = key[16:pos] # 16 = len('operator family ') + idx = key[pos + 7:] # 7 = len(' using ') + inobj = inopfams[key] + self[(schema.name, opf, idx)] = OperatorFamily.from_map( + opf, schema, idx, inobj) + + def find(self, sch, obj, meth): + schema, name = split_schema_obj(obj, sch) + return self.get((schema, name, meth)) diff --git a/pyrseas/dbobject/privileges.py b/pyrseas/dbobject/privileges.py new file mode 100644 index 0000000..4c9b48c --- /dev/null +++ b/pyrseas/dbobject/privileges.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.privileges + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines functions for dealing with access privileges. +""" + +PRIVCODES = {'a': 'insert', 'r': 'select', 'w': 'update', 'd': 'delete', + 'D': 'truncate', 'x': 'references', 't': 'trigger', + 'X': 'execute', 'U': 'usage', 'C': 'create'} +PRIVILEGES = dict((v, k) for k, v in list(PRIVCODES.items())) + + +def _split_privs(privspec): + """Split the aclitem into three parts + + :param privspec: privilege specification (aclitem) + :return: tuple with grantee, privilege codes and granto + """ + (usr, prvgrant) = privspec.split('=') + if usr == '': + usr = 'PUBLIC' + (privcodes, grantor) = prvgrant.split('/') + return (usr, privcodes, grantor) + + +def _expand_priv_lists(obj, privcodes, subobj): + """Convert privilege code strings to expanded lists + + :param obj: the object on which the privilege is granted + :param privcodes: string of privilege codes + :param subobj: sub-object name (e.g., column name) + :return: tuple of lists with decoded privileges + """ + privs = [] + wgo = [] + if privcodes == obj.allprivs and len(obj.allprivs) > 1: + privs = ['ALL'] + else: + if subobj: + subobj = ' (%s)' % subobj + for code in sorted(PRIVCODES.keys()): + if code in privcodes: + priv = PRIVCODES[code].upper() + subobj + if code + '*' in privcodes: + wgo.append(priv) + else: + privs.append(priv) + return (privs, wgo) + + +def privileges_to_map(privspec, allprivs, owner): + """Map a set of privileges in PostgreSQL format to YAML-suitable format + + :param privspec: privilege specification + :param allprivs: privilege list equal to ALL + :param owner: object owner + :return: dictionary + + Access privileges are specified as aclitem's as follows: + =/. The grantee and grantor are user + names. The privlist is a set of single letter codes, each letter + optionally followed by an asterisk to indicate WITH GRANT OPTION. + """ + (usr, privcodes, grantor) = _split_privs(privspec) + privs = [] + if privcodes == allprivs and len(allprivs) > 1: + privs = ['all'] + else: + for code in sorted(PRIVCODES.keys()): + if code in privcodes: + priv = PRIVCODES[code] + if code + '*' in privcodes: + priv = {priv: {'grantable': True}} + privs.append(priv) + if owner and grantor != owner: + privs = {'privs': privs, 'grantor': grantor} + return {usr: privs} + + +def privileges_from_map(privlist, allprivs, owner): + """Map privileges from YAML-suitable format to an internal list + + :param privspec: privilege specification + :param allprivs: privilege list equal to ALL + :param owner: object owner + :return: list + """ + retlist = [] + for priv in privlist: + usr = list(priv.keys())[0] + privs = priv[usr] + grantor = owner + if 'grantor' in privs: + grantor = privs['grantor'] + privs = privs['privs'] + if usr == 'PUBLIC': + usr = '' + prvcodes = '' + if privs == ['all']: + prvcodes = allprivs + else: + for prv in privs: + if isinstance(prv, dict): + key = list(prv.keys())[0] + else: + key = prv + if key in PRIVILEGES: + prvcodes += PRIVILEGES[key] + if isinstance(prv, dict) and isinstance(prv[key], dict) and \ + 'grantable' in prv[key] and prv[key]['grantable']: + prvcodes += '*' + retlist.append("%s=%s/%s" % (usr, prvcodes, grantor)) + return retlist + + +def add_grant(obj, privspec, subobj=''): + """Return GRANT statements on the object based on the privilege spec + + :param obj: the object on which the privilege is granted + :param privspec: the privilege specification (aclitem) + :param subobj: sub-object name (e.g., column name) + :return: list of GRANT statements + """ + (usr, privcodes, grantor) = _split_privs(privspec) + (privs, wgo) = _expand_priv_lists(obj, privcodes, subobj) + objtype = obj.objtype + if hasattr(obj, 'privobjtype'): + objtype = obj.privobjtype + stmts = [] + if privs: + stmts.append("GRANT %s ON %s %s TO %s" % ( + ', '.join(privs), objtype, obj.identifier(), usr)) + if wgo: + stmts.append("GRANT %s ON %s %s TO %s WITH GRANT OPTION" % ( + ', '.join(wgo), objtype, obj.identifier(), usr)) + return stmts + + +def add_revoke(obj, privspec, subobj=''): + """Return REVOKE statements on the object based on the privilege spec + + :param obj: the object on which the privilege is to be revoked + :param privspec: the privilege specification (aclitem) + :param subobj: sub-object name (e.g., column name) + :return: list of REVOKE statements + """ + (usr, privcodes, grantor) = _split_privs(privspec) + (privs, wgo) = _expand_priv_lists(obj, privcodes, subobj) + objtype = obj.objtype + if hasattr(obj, 'privobjtype'): + objtype = obj.privobjtype + stmts = [] + if wgo: + stmts.append("REVOKE %s ON %s %s FROM %s" % ( + ', '.join(wgo), objtype, obj.identifier(), usr)) + if privs: + stmts.append("REVOKE %s ON %s %s FROM %s" % ( + ', '.join(privs), objtype, obj.identifier(), usr)) + return stmts + + +def diff_privs(currobj, currlist, newobj, newlist, subobj=''): + """Return GRANT or REVOKE statements to adjust object privileges + + :param currobj: current object + :param currlist: list of current privileges + :param newobj: new object + :param newlist: list of new privileges + :param subobj: sub-object (e.g., column name) + :return: list of GRANT and REVOKE statements + """ + def rejoin(privdict, usr, grantor): + return '%s=%s/%s' % ('' if usr == 'PUBLIC' else usr, + privdict[(usr, grantor)], grantor) + + stmts = [] + currprivs = {} + newprivs = {} + for privspec in currlist: + if privspec: + (usr, privcodes, grantor) = _split_privs(privspec) + currprivs[(usr, grantor)] = privcodes + for privspec in newlist: + if privspec: + (usr, privcodes, grantor) = _split_privs(privspec) + newprivs[(usr, grantor)] = privcodes + for (usr, gtor) in currprivs: + if (usr, gtor) not in newprivs: + stmts.append(add_revoke(currobj, rejoin(currprivs, usr, gtor), + subobj)) + for (usr, gtor) in newprivs: + if (usr, gtor) not in currprivs: + stmts.append(add_grant(newobj, rejoin(newprivs, usr, gtor), + subobj)) + else: + if sorted(currprivs[(usr, gtor)]) != sorted(newprivs[(usr, gtor)]): + stmts.append(add_revoke(currobj, rejoin(currprivs, usr, gtor), + subobj)) + stmts.append(add_grant(newobj, rejoin(newprivs, usr, gtor), + subobj)) + return stmts diff --git a/pyrseas/dbobject/rule.py b/pyrseas/dbobject/rule.py new file mode 100644 index 0000000..9f50b60 --- /dev/null +++ b/pyrseas/dbobject/rule.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.rule + ~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Rule and RuleDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, split_schema_obj + + +class Rule(DbSchemaObject): + """A rewrite rule definition""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_rewrite' + + def __init__(self, name, schema, table, description, event, instead=False, + actions=None, condition=None, definition=None, + oid=None): + """Initialize the rewrite rule + + :param name: rule name (from rulename) + :param schema: schema name (from nspname via relnamespace/ev_class) + :param table: table name (from relname via ev_class) + :param description: comment text (from obj_description()) + :param event: event type (from ev_type) + :param instead: is it an INSTEAD rule? (from is_instead) + :param actions: rule actions (from ev_action via definition) + :param condition: qualifying condition (from ev_qual via definition) + :param definition: "raw" definition (from pg_get_ruledef) + """ + super(Rule, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = table + self.description = description + self.event = event + self.instead = instead + if definition is not None: + assert actions is None + assert condition is None + do_loc = definition.index(' DO ') + if 'WHERE' in definition: + self.condition = definition[ + definition.index(' WHERE ') + 7:do_loc] + if instead: + do_loc += 8 + self.actions = definition[do_loc + 4:-1] + else: + self.actions = actions + self.condition = condition + self.oid = oid + + @staticmethod + def query(dbversion=None): + """Returns query to fetch Rule instances from the catalogs""" + return """ + SELECT nspname AS schema, relname AS table, rulename AS name, + split_part('select,update,insert,delete', ',', + ev_type::int - 48) AS event, is_instead AS instead, + pg_get_ruledef(r.oid) AS definition, + obj_description(r.oid, 'pg_rewrite') AS description, r.oid + FROM pg_rewrite r JOIN pg_class c ON (ev_class = c.oid) + JOIN pg_namespace n ON (relnamespace = n.oid) + WHERE relkind = 'r' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, relname, rulename""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a Rule instance from a YAML map + + :param name: rule name + :param table: map of the table associated with the rule + :param inobj: YAML map of the rule + :return: Rule instance + """ + obj = Rule( + name, table.schema, table.name, inobj.pop('description', None), + inobj.get('event'), inobj.pop('instead', False), + inobj.pop('actions', None), inobj.pop('condition', None)) + obj.set_oldname(inobj) + return obj + + def identifier(self): + """Return a full identifier for a rule object + + :return: string + """ + return "%s ON %s" % (quote_id(self.name), self._table.qualname()) + + def to_map(self, db): + """Convert rule to a YAML-suitable format + + :return: dictionary + """ + dct = super(Rule, self).to_map(db) + if not self.instead: + dct.pop('instead') + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the rule + + :return: SQL statements + """ + where = instead = '' + if self.condition is not None: + where = ' WHERE %s' % self.condition + if self.instead: + instead = 'INSTEAD ' + return ["CREATE RULE %s AS ON %s\n TO %s%s\n DO %s%s" % ( + quote_id(self.name), self.event.upper(), + self._table.qualname(), where, instead, self.actions)] + + def get_implied_deps(self, db): + """Return set of implicit dependencies + + :param db: Database.Dicts object + :return: set of dependencies + """ + deps = super(Rule, self).get_implied_deps(db) + deps.add(db.tables[split_schema_obj(self.table, self.schema)]) + return deps + + +class RuleDict(DbObjectDict): + "The collection of rewrite rules in a database." + + cls = Rule + + def from_map(self, table, inmap): + """Initialize the dictionary of rules by examining the input map + + :param table: the input YAML map of the associated table + :param inmap: the input YAML map defining the rules + """ + for rul in inmap: + inobj = inmap[rul] + self[(table.schema, table.name, rul)] = Rule.from_map( + rul, table, inobj) diff --git a/pyrseas/dbobject/schema.py b/pyrseas/dbobject/schema.py new file mode 100644 index 0000000..9f5d767 --- /dev/null +++ b/pyrseas/dbobject/schema.py @@ -0,0 +1,371 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.schema + ~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Schema and SchemaDict, derived from + DbObject and DbObjectDict, respectively. +""" +import os + +from pyrseas.yamlutil import yamldump +from . import DbObjectDict, DbObject +from . import quote_id, commentable, ownable, grantable +from .dbtype import BaseType, Composite, Domain, Enum, Range +from .table import Table, Sequence +from .view import View, MaterializedView + + +class Schema(DbObject): + """A database schema definition, i.e., a named collection of tables, + views, triggers and other schema objects.""" + + keylist = ['name'] + catalog = 'pg_namespace' + + @property + def allprivs(self): + return 'UC' + + def __init__(self, name, description=None, owner=None, privileges=[], + oldname=None, oid=None): + """Initialize the schema + + :param name: schema name (from nspname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via nspowner) + :param privileges: access privileges (from nspacl) + :param oldname: previous name of schema + """ + super(Schema, self).__init__(name, description) + self._init_own_privs(owner, privileges) + self.oldname = None + + @staticmethod + def query(dbversion=None): + return r""" + SELECT nspname AS name, rolname AS owner, + array_to_string(nspacl, ',') AS privileges, + obj_description(n.oid, 'pg_namespace') AS description, n.oid + FROM pg_namespace n + JOIN pg_roles r ON (r.oid = nspowner) + WHERE nspname NOT IN ('information_schema', 'pg_toast') + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND n.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_namespace'::regclass) + ORDER BY nspname""" + + @staticmethod + def from_map(name, inobj): + """Initialize a schema instance from a YAML map + + :param name: schema name + :param inobj: YAML map of the schema + :return: schema instance + """ + obj = Schema( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('privileges', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def extern_dir(self, root='.'): + """Return the path to a directory to hold the schema objects. + + :return: directory path + """ + (dir, ext) = os.path.splitext(os.path.join(root, + self.extern_filename())) + return dir + + def to_map(self, db, dbschemas, opts): + """Convert tables, etc., dictionaries to a YAML-suitable format + + :param dbschemas: dictionary of schemas + :param opts: options to include/exclude schemas/tables, etc. + :return: dictionary + """ + if self.name == 'pyrseas': + return {} + no_owner = opts.no_owner + no_privs = opts.no_privs + schbase = {} if no_owner else {'owner': self.owner} + if not no_privs: + schbase.update({'privileges': self.map_privs()}) + if self.description is not None: + schbase.update(description=self.description) + + schobjs = [] + seltbls = getattr(opts, 'tables', []) + if hasattr(self, 'tables'): + for objkey in self.tables: + if not seltbls or objkey in seltbls: + obj = self.tables[objkey] + schobjs.append((obj, obj.to_map(db, dbschemas, opts))) + + def mapper(objtypes): + if hasattr(self, objtypes): + schemadict = getattr(self, objtypes) + for objkey in schemadict: + if objtypes == 'sequences' or ( + not seltbls or objkey in seltbls): + obj = schemadict[objkey] + schobjs.append((obj, obj.to_map(db, opts))) + + for objtypes in ['ftables', 'sequences', 'views', 'matviews']: + mapper(objtypes) + + def mapper2(objtypes, privs=False): + if hasattr(self, objtypes): + schemadict = getattr(self, objtypes) + for objkey in schemadict: + obj = schemadict[objkey] + if privs: + dct = obj.to_map(db, no_owner, no_privs) + else: + dct = obj.to_map(db, no_owner) + schobjs.append((obj, dct)) + + if hasattr(opts, 'tables') and not opts.tables or \ + not hasattr(opts, 'tables'): + for objtypes in ('conversions', 'collations', 'operators', + 'operclasses', 'operfams', 'tsconfigs', + 'tsdicts', 'tsparsers', 'tstempls'): + mapper2(objtypes) + for objtypes in ('types', 'domains'): + mapper2(objtypes, True) + if hasattr(self, 'functions'): + for objkey in self.functions: + obj = self.functions[objkey] + schobjs.append((obj, obj.to_map(db, no_owner, no_privs))) + + # special case for pg_catalog schema + if self.name == 'pg_catalog' and not schobjs: + return {} + + if hasattr(self, 'datacopy') and self.datacopy: + dir = self.extern_dir(opts.data_dir) + if not os.path.exists(dir): + os.mkdir(dir) + for tbl in self.datacopy: + self.tables[tbl].data_export(dbschemas.dbconn, dir) + dbschemas.dbconn.commit() + + if opts.multiple_files: + dir = self.extern_dir(opts.metadata_dir) + if not os.path.exists(dir): + os.mkdir(dir) + filemap = {} + for obj, objmap in schobjs: + if objmap is not None: + extkey = obj.extern_key() + filepath = os.path.join(dir, obj.extern_filename()) + with open(filepath, 'a') as f: + f.write(yamldump({extkey: objmap})) + outobj = {extkey: + os.path.relpath(filepath, opts.metadata_dir)} + filemap.update(outobj) + # always write the schema YAML file + filepath = self.extern_filename() + extkey = self.extern_key() + with open(os.path.join(opts.metadata_dir, filepath), 'a') as f: + f.write(yamldump({extkey: schbase})) + filemap.update(schema=filepath) + return {extkey: filemap} + + schmap = dict((obj.extern_key(), objmap) for obj, objmap in schobjs + if objmap is not None) + schmap.update(schbase) + return {self.extern_key(): schmap} + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the schema + + :return: SQL statements + """ + # special case for --revert + if self.name == 'pg_catalog': + return [] + return ["CREATE SCHEMA %s" % quote_id(self.name)] + + def data_import(self, opts): + """Generate SQL to import data from the tables in this schema + + :param opts: options to include/exclude schemas/tables, etc. + :return: list of SQL statements + """ + stmts = [] + if hasattr(self, 'datacopy') and self.datacopy: + dir = self.extern_dir(opts.data_dir) + for tbl in self.datacopy: + stmts.append(self.tables[tbl].data_import(dir)) + return stmts + + def drop(self): + if self.name not in ('public', 'pg_catalog'): + return super(Schema, self).drop() + else: + return [] + + +PREFIXES = {'domain ': 'types', 'type': 'types', 'table ': 'tables', + 'view ': 'tables', 'sequence ': 'tables', + 'materialized view ': 'tables', + 'function ': 'functions', 'aggregate ': 'functions', + 'operator family ': 'operfams', 'operator class ': 'operclasses', + 'conversion ': 'conversions', 'text search dictionary ': 'tsdicts', + 'text search template ': 'tstempls', + 'text search parser ': 'tsparsers', + 'text search configuration ': 'tsconfigs', + 'foreign table ': 'ftables', 'collation ': 'collations'} +SCHOBJS1 = ['types', 'tables', 'ftables'] +SCHOBJS2 = ['collations', 'conversions', 'functions', 'operators', + 'operclasses', 'operfams', 'tsconfigs', 'tsdicts', 'tsparsers', + 'tstempls'] + + +class SchemaDict(DbObjectDict): + "The collection of schemas in a database. Minimally, the 'public' schema." + + cls = Schema + + def from_map(self, inmap, newdb): + """Initialize the dictionary of schemas by converting the input map + + :param inmap: the input YAML map defining the schemas + :param newdb: collection of dictionaries defining the database + + Starts the recursive analysis of the input map and + construction of the internal collection of dictionaries + describing the database objects. + """ + for key in inmap: + (objtype, spc, sch) = key.partition(' ') + if spc != ' ' or objtype != 'schema': + raise KeyError("Unrecognized object type: %s" % key) + inobj = inmap[key] + schema = self[sch] = Schema.from_map(sch, inobj) + objdict = {} + for key in sorted(inobj.keys()): + mapped = False + for prefix in PREFIXES: + if key.startswith(prefix): + otype = PREFIXES[prefix] + if otype not in objdict: + objdict[otype] = {} + objdict[otype].update({key: inobj[key]}) + mapped = True + break + # Needs separate processing because it overlaps + # operator classes and operator families + if not mapped and key.startswith('operator '): + otype = 'operators' + if otype not in objdict: + objdict[otype] = {} + objdict[otype].update({key: inobj[key]}) + mapped = True + elif key == 'oldname': + inobj.pop(key) + mapped = True + if not mapped and key != 'schema': + raise KeyError("Expected typed object, found '%s'" % key) + + for objtype in SCHOBJS1: + if objtype in objdict: + subobjs = getattr(newdb, objtype) + subobjs.from_map(schema, objdict[objtype], newdb) + for objtype in SCHOBJS2: + if objtype in objdict: + subobjs = getattr(newdb, objtype) + subobjs.from_map(schema, objdict[objtype]) + + def link_refs(self, db, datacopy): + """Connect various schema objects to their respective schemas + + :param db: dictionary of dictionaries of all objects + :param datacopy: dictionary of data copying info + """ + def link_one(targdict, objtype, objkeys, subtype=None): + schema = self[objkeys[0]] + if subtype is None: + subtype = objtype + if not hasattr(schema, subtype): + setattr(schema, subtype, {}) + objdict = getattr(schema, subtype) + key = objkeys[1] if len(objkeys) == 2 else objkeys[1:] + objdict.update({key: targdict[objkeys]}) + + targ = db.types + for keys in targ: + dbtype = targ[keys] + if isinstance(dbtype, Domain): + link_one(targ, 'types', keys, 'domains') + elif isinstance(dbtype, (Enum, Composite, BaseType, Range)): + link_one(targ, 'types', keys) + targ = db.tables + for keys in targ: + table = targ[keys] + type_ = 'tables' + if isinstance(table, Table): + link_one(targ, type_, keys) + elif isinstance(table, Sequence): + link_one(targ, type_, keys, 'sequences') + elif isinstance(table, MaterializedView): + link_one(targ, type_, keys, 'matviews') + elif isinstance(table, View): + link_one(targ, type_, keys, 'views') + targ = db.functions + for keys in targ: + link_one(targ, 'functions', keys) + for objtype in ['operators', 'operclasses', 'operfams', 'conversions', + 'tsconfigs', 'tsdicts', 'tsparsers', 'tstempls', + 'ftables', 'collations']: + targ = getattr(db, objtype) + for keys in targ: + link_one(targ, objtype, keys) + for key in datacopy: + if not key.startswith('schema '): + raise KeyError("Unrecognized object type: %s" % key) + sch = key[7:] + if sch not in self: + continue + schema = self[sch] + if not hasattr(schema, 'datacopy'): + schema.datacopy = [] + for tbl in datacopy[key]: + if hasattr(schema, 'tables') and tbl in schema.tables: + schema.datacopy.append(tbl) + + def to_map(self, db, opts): + """Convert the schema dictionary to a regular dictionary + + :param opts: options to include/exclude schemas/tables, etc. + :return: dictionary + + Invokes the `to_map` method of each schema to construct a + dictionary of schemas. + """ + schemas = {} + selschs = getattr(opts, 'schemas', []) + for sch in self: + if not selschs or sch in selschs: + if hasattr(opts, 'excl_schemas') and opts.excl_schemas \ + and sch in opts.excl_schemas: + continue + schemas.update(self[sch].to_map(db, self, opts)) + + return schemas + + def data_import(self, opts): + """Iterate over schemas with tables to be imported + + :param opts: options to include/exclude schemas/tables, etc. + :return: list of SQL statements + """ + return [self[sch].data_import(opts) for sch in self] diff --git a/pyrseas/dbobject/table.py b/pyrseas/dbobject/table.py new file mode 100644 index 0000000..66a49d9 --- /dev/null +++ b/pyrseas/dbobject/table.py @@ -0,0 +1,985 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.table + ~~~~~~~~~~~~~~~~~~~~~~ + + This module defines four classes: DbClass derived from + DbSchemaObject, Sequence and Table derived from DbClass, and + ClassDict derived from DbObjectDict. +""" + +import copy +import re +import os +import sys + +from . import DbObjectDict, DbSchemaObject, split_schema_obj +from . import quote_id, commentable, ownable, grantable +from .constraint import CheckConstraint, PrimaryKey +from .constraint import ForeignKey, UniqueConstraint +from .privileges import add_grant + +MAX_BIGINT = 9223372036854775807 + + +def seq_max_value(seq): + if seq.max_value is None or seq.max_value == MAX_BIGINT: + return " NO MAXVALUE" + return " MAXVALUE %d" % seq.max_value + + +def seq_min_value(seq): + if seq.min_value is None or seq.min_value == 1: + return " NO MINVALUE" + return " MINVALUE %d" % seq.min_value + + +class DbClass(DbSchemaObject): + """A table, sequence or view + + The `pg_class` catalog also includes Postgres indexes, but for now, + indexes have not been implemented as part of the `DbClass` hierarchy. + """ + + keylist = ['schema', 'name'] + catalog = 'pg_class' + + def __init__(self, name, schema, description, owner, privileges): + """Initialize the relation "class" + + :param name: relation name (from relname) + :param schema: schema name (from relnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via relowner) + :param privileges: access privileges (from relacl) + """ + super(DbClass, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + + +class Sequence(DbClass): + "A sequence generator definition" + + def __init__(self, name, schema, description, owner, privileges, + start_value=1, increment_by=1, max_value=MAX_BIGINT, + min_value=1, cache_value=1, data_type='bigint', + owner_table=None, owner_column=None, + oid=None): + """Initialize the sequence + + :param name-privileges: see DbClass.__init__ params + :param start_value: start value (from start_value) + :param max_value: maximum value (from max_value) + :param increment_by: value to add (from increment_by) + :param min_value: minimum value (from min_value) + :param cache_value: cache value (from cache_value/cache_size) + :param data_type: data type (from data_type) + :param owner_table: owner table + :param owner_column: owner column + """ + super(Sequence, self).__init__(name, schema, description, owner, + privileges) + self.start_value = start_value + self.increment_by = increment_by + self.max_value = max_value + self.min_value = min_value + self.cache_value = cache_value + self.data_type = data_type + self.owner_table = owner_table + self.owner_column = owner_column + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'S' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + AND c.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_class'::regclass) + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a sequence instance from a YAML map + + :param name: sequence name + :param name: schema map + :param inobj: YAML map of the sequence + :return: sequence instance + """ + obj = Sequence( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('start_value', 1), inobj.pop('increment_by', 1), + inobj.pop('max_value', MAX_BIGINT), inobj.pop('min_value', 1), + inobj.pop('cache_value', 1), inobj.pop('data_type', 'bigint'), + inobj.pop('owner_table', None), inobj.pop('owner_column', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def allprivs(self): + return 'rwU' + + def get_attrs(self, dbconn): + """Get the attributes for the sequence + + :param dbconn: a DbConnection object + """ + query = """SELECT start_value, increment_by, max_value, min_value, + cache_size AS cache_value, data_type + FROM pg_sequences + WHERE schemaname = '%s' + AND sequencename = '%s'""" % (self.schema, self.name) + data = dbconn.fetchone(query) + + for key, val in list(data.items()): + setattr(self, key, val) + + def get_dependent_table(self, dbconn): + """Get the table and column name that uses or owns the sequence + + :param dbconn: a DbConnection object + """ + + def split_table(obj, sch): + schema = sch + tbl = obj + quoted = '"%s".' % schema + if obj.startswith(schema + '.'): + tbl = obj[len(schema) + 1:] + elif obj.startswith(quoted): + tbl = obj[len(quoted):] + elif sch is None: + raise ValueError("Invalid schema.table: %s" % obj) + if tbl[0] == '"' and tbl[-1:] == '"': + tbl = tbl[1:-1] + return tbl + + data = dbconn.fetchone( + """SELECT refobjid::regclass, refobjsubid + FROM pg_depend + WHERE objid = '%s'::regclass + AND refclassid = 'pg_class'::regclass""" % self.identifier()) + if data: + self.owner_table = split_table(data["refobjid"], self.schema) + self.owner_column = data["refobjsubid"] + return + data = dbconn.fetchone( + """SELECT adrelid::regclass AS regclass + FROM pg_attrdef a JOIN pg_depend ON (a.oid = objid) + WHERE refobjid = '%s'::regclass + AND classid = 'pg_attrdef'::regclass""" % self.qualname()) + if data: + self.dependent_table = split_table(data["regclass"], self.schema) + + def to_map(self, db, opts): + """Convert a sequence definition to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'tables') and opts.tables and \ + (self.name not in opts.tables and + self.owner_table is None or + self.owner_table not in opts.tables) or ( + hasattr(opts, 'excl_tables') and opts.excl_tables and + self.name in opts.excl_tables): + return None + dct = super(Sequence, self).to_map(db, opts.no_owner, opts.no_privs) + if self.owner_table is None and self.owner_column is None: + dct.pop('owner_table') + dct.pop('owner_column') + dct.pop('dependent_table', None) + if self.data_type == 'bigint': + dct.pop('data_type') + for key, val in list(dct.items()): + if key == 'max_value' and val == MAX_BIGINT: + dct[key] = None + elif key == 'min_value' and val == 1: + dct[key] = None + elif key == 'privileges': + dct[key] = val + else: + if isinstance(val, int): + dct[key] = int(val) + else: + dct[key] = str(val) + + return dct + + def _common_create_spec(self): + return """ START WITH %d + INCREMENT BY %d + %s + %s + CACHE %d""" % (self.start_value, self.increment_by, seq_min_value(self), + seq_max_value(self), self.cache_value) + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return a SQL statement to CREATE the sequence + + :return: SQL statements + """ + mod = "" + if self.data_type != 'bigint': + mod = "\n AS %s" % self.data_type + if hasattr(self, '_owner_col'): + if self._owner_col.identity is not None: + return [] + return ["CREATE SEQUENCE %s%s%s" % (self.qualname(), mod, + self._common_create_spec())] + + def add_inline(self): + """Return statement to create the sequence inline as part of a + GENERATED AS IDENTITY clause + """ + return "SEQUENCE NAME %s%s" % (self.qualname(), + self._common_create_spec()) + + def add_owner(self): + """Return statement to ALTER the sequence to indicate its owner table + + :return: SQL statement + """ + stmts = [] + stmts.append("ALTER SEQUENCE %s OWNED BY %s.%s" % ( + self.qualname(), self.qualname(self.schema, self.owner_table), + quote_id(self.owner_column))) + return stmts + + def alter(self, inseq, no_owner=False): + """Generate SQL to transform an existing sequence + + :param inseq: a YAML map defining the new sequence + :return: list of SQL statements + + Compares the sequence to an input sequence and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + stmt = "" + if self.start_value != inseq.start_value: + stmt += " START WITH %d" % inseq.start_value + if self.increment_by != inseq.increment_by: + stmt += " INCREMENT BY %d" % inseq.increment_by + minval = self.min_value + if minval == 1: + minval = None + if minval != inseq.min_value: + stmt += seq_min_value(inseq) + maxval = self.max_value + if maxval == MAX_BIGINT: + maxval = None + if maxval != inseq.max_value: + stmt += seq_max_value(inseq) + if self.cache_value != inseq.cache_value: + stmt += " CACHE %d" % inseq.cache_value + if stmt: + stmts.append("ALTER SEQUENCE %s" % self.qualname() + stmt) + + if inseq.owner_column is not None and inseq.owner_table is None: + raise ValueError("Sequence '%s' incomplete specification: " + "owner_column but no owner_table") + if inseq.owner_table is not None: + if inseq.owner_column is None: + raise ValueError("Sequence '%s' incomplete specification: " + "owner_table but no owner_column") + if self.owner_table is None and self.owner_column is None: + stmts.append(inseq.add_owner()) + + stmts.append(super(Sequence, self).alter(inseq, no_owner=no_owner)) + return stmts + + def drop(self): + """Generate SQL to drop the current sequence + + :return: list of SQL statements + """ + stmts = [] + if self.owner_table is None: + stmts.append(super(Sequence, self).drop()) + return stmts + + +PARTITIONING_STRATEGIES = {'l': 'list', 'r': 'range'} + + +class Table(DbClass): + """A database table definition + + A table is identified by its schema name and table name. It should + have a list of columns. It may have a primary_key, zero or more + foreign_keys, zero or more unique_constraints, and zero or more + indexes. + + A :class:`Table` can also represent a partitioned table or a + partition of a partitioned table. The latter's columns are all + inherited from the parent (partitioned) table, so they are not + shown in an output map (or expected on input). + """ + def __init__(self, name, schema, description, owner, privileges, + tablespace=None, unlogged=False, options=None, + partition_bound_spec=None, partition_by=None, + partition_cols=None, partition_exprs=None, + oid=None): + """Initialize the table + + :param name-privileges: see DbClass.__init__ params + :param tablespace: storage tablespace (from reltablespace) + :param unlogged: unlogged indicator (from relpersistence = 'u') + :param options: access method options (from reloptions) + :param partition_bound_spec: partition bound (from relpartbound) + :param partition_by: partitioning strategy (from partstrat) + """ + super(Table, self).__init__(name, schema, description, owner, + privileges) + self.tablespace = tablespace + self.unlogged = unlogged + self.options = options + if partition_bound_spec is None: + self.partition_bound_spec = None + elif partition_bound_spec.startswith("FOR VALUES "): + self.partition_bound_spec = partition_bound_spec[11:] + else: + self.partition_bound_spec = partition_bound_spec + if partition_by is None: + self.partition_by = None + elif isinstance(partition_by, dict): + partby = list(partition_by.keys())[0] + self.partition_by = partby + self.partition_cols = partition_by[partby] + elif len(partition_by) == 1: + self.partition_by = PARTITIONING_STRATEGIES[partition_by] + else: + assert partition_by in PARTITIONING_STRATEGIES.values() + self.partition_by = partition_by + if partition_cols is not None and len(partition_cols) > 0: + self.partition_cols = [int(n) for n in partition_cols.split(' ')] + elif partition_cols is None and partition_by is None: + self.partition_cols = partition_cols + self.partition_exprs = partition_exprs + self.columns = [] + self.inherits = [] + self.check_constraints = {} + self.primary_key = None + self.foreign_keys = {} + self.unique_constraints = {} + self.indexes = {} + self.triggers = {} + self.rules = {} + self.oid = oid + self._referred_by = [] + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, reloptions AS options, + spcname AS tablespace, relpersistence = 'u' AS unlogged, + rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_expr(relpartbound, c.oid) AS partition_bound_spec, + partstrat AS partition_by, partattrs AS partition_cols, + pg_get_expr(partexprs, pt.partrelid) AS partition_exprs, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + LEFT JOIN pg_tablespace t ON (reltablespace = t.oid) + LEFT JOIN pg_partitioned_table pt ON c.oid = pt.partrelid + WHERE relkind IN ('r', 'p') AND relpersistence != 't' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + AND c.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_class'::regclass) + ORDER BY nspname, relname""" + + @staticmethod + def inhquery(): + return """SELECT inhrelid::regclass AS sub, + inhparent::regclass AS parent, inhseqno + FROM pg_inherits + ORDER BY 1, 3""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a table instance from a YAML map + + :param name: table name + :param name: schema map + :param inobj: YAML map of the table + :return: table instance + """ + obj = Table( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('tablespace', None), inobj.pop('unlogged', False), + inobj.pop('options', None), + inobj.pop('partition_bound_spec', None), + inobj.pop('partition_by', None)) + if obj.partition_bound_spec is not None: + obj.inherits = [inobj.get('partition_of')] + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def allprivs(self): + return 'arwdDxt' + + def _normalize_partcols(self): + "Replace integer column numbers by column names" + if isinstance(self.partition_cols[0], int): + self.partition_cols = [self.columns[k - 1].name + for k in self.partition_cols] + + def column_names(self): + """Return a list of column names in the table + + :return: list + """ + return [c.name for c in self.columns] + + def to_map(self, db, dbschemas, opts): + """Convert a table to a YAML-suitable format + + :param dbschemas: database dictionary of schemas + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables or len(self.columns) == 0: + return None + + dct = super(Table, self).to_map(db, opts.no_owner, opts.no_privs, + deepcopy=False) + + for attr in ('tablespace', 'options', 'partition_bound_spec'): + if dct[attr] is None: + dct.pop(attr) + if self.unlogged is False: + dct.pop('unlogged') + if len(self.inherits) == 0: + dct.pop('inherits') + + if self.partition_bound_spec is None: + cols = [] + for column in self.columns: + col = column.to_map(db, opts.no_privs) + if col: + cols.append(col) + dct['columns'] = cols + else: + dct.pop('columns') + assert len(self.inherits) == 1 + dct.update(partition_of=self.inherits[0]) + dct.pop('inherits') + + if self.partition_by is not None: + assert self.partition_cols is not None + self._normalize_partcols() + dct.update(partition_by={self.partition_by: self.partition_cols}) + else: + dct.pop('partition_by') + dct.pop('partition_cols') + dct.pop('partition_exprs') + + if len(self.check_constraints) > 0: + dct['check_constraints'] = copy.copy(dct['check_constraints']) + for k in list(self.check_constraints.values()): + dct['check_constraints'].update( + self.check_constraints[k.name].to_map( + db, self.column_names())) + else: + dct.pop('check_constraints') + if self.primary_key is not None: + dct['primary_key'] = self.primary_key.to_map( + db, self.column_names()) + else: + dct.pop('primary_key') + if len(self.foreign_keys) > 0: + dct['foreign_keys'] = copy.copy(dct['foreign_keys']) + for k in list(self.foreign_keys.values()): + tbls = dbschemas[k.ref_schema].tables + ktable = self.foreign_keys[k.name] + dct['foreign_keys'].update(ktable.to_map( + db, self.column_names(), + tbls[ktable.ref_table].column_names())) + else: + dct.pop('foreign_keys') + if len(self.unique_constraints) > 0: + dct['unique_constraints'] = copy.copy(dct['unique_constraints']) + for k in list(self.unique_constraints.values()): + dct['unique_constraints'].update( + self.unique_constraints[k.name].to_map( + db, self.column_names())) + else: + dct.pop('unique_constraints') + if len(self.indexes) > 0: + idxs = {} + for idx in self.indexes.values(): + if not getattr(idx, '_for_constraint', None): + idxs.update(idx.to_map(db)) + if idxs: + # we may have only indexes not to dump, e.g. the pkey one + dct['indexes'] = idxs + else: + dct.pop('indexes', None) + else: + dct.pop('indexes') + if len(self.rules) > 0: + dct['rules'] = copy.copy(dct['rules']) + for k in list(self.rules.values()): + dct['rules'].update(self.rules[k.name].to_map(db)) + else: + dct.pop('rules') + if len(self.triggers) > 0: + dct['triggers'] = copy.copy(dct['triggers']) + for k in list(self.triggers.values()): + dct['triggers'].update(self.triggers[k.name].to_map(db)) + else: + dct.pop('triggers') + + return copy.deepcopy(dct) + + def create(self, dbversion=None): + """Return SQL statements to CREATE the table + + :return: SQL statements + """ + stmts = [] + cols = [] + colprivs = [] + for col in self.columns: + if not col.inherited: + cols.append(" " + col.add()[0]) + colprivs.append(col.add_privs()) + unlogged = 'UNLOGGED ' if self.unlogged else '' + + partbyclause = partofclause = inhclause = collist = "" + if self.partition_by is not None: + partbyclause = " PARTITION BY %s (%s)" % ( + self.partition_by.upper(), ", ".join(self.partition_cols)) + elif len(self.inherits) > 0: + inhclause = " INHERITS (%s)" % ", ".join( + self.qualname(self.schema, t) for t in self.inherits) + if self.partition_bound_spec is None: + collist = "(\n%s)" % ",\n".join(cols) + else: + partofclause = "PARTITION OF %s FOR VALUES %s" % ( + self.inherits[0], self.partition_bound_spec) + inhclause = "" + + opts = '' + if self.options is not None: + opts = " WITH (%s)" % ', '.join(self.options) + tblspc = '' + if self.tablespace is not None: + tblspc = " TABLESPACE %s" % self.tablespace + stmts.append("CREATE %sTABLE %s %s%s%s%s%s%s" % ( + unlogged, self.qualname(), collist, partbyclause, partofclause, + inhclause, opts, tblspc)) + if self.owner is not None: + stmts.append(self.alter_owner()) + for priv in self.privileges: + stmts.append(add_grant(self, priv)) + if colprivs: + stmts.append(colprivs) + if self.description is not None: + stmts.append(self.comment()) + for col in self.columns: + if col.description is not None: + stmts.append(col.comment()) + if hasattr(self, '_owned_seqs'): + for dep in self._owned_seqs: + stmts.append(dep.add_owner()) + self.created = True + return stmts + + def drop(self): + """Return a SQL DROP statement for the table + + :return: SQL statement + """ + stmts = [] + if not hasattr(self, 'dropped') or not self.dropped: + if hasattr(self, '_dependent_funcs'): + for fnc in self._dependent_funcs: + stmts.append(fnc.drop()) + self.dropped = True + stmts.append("DROP TABLE %s" % self.identifier()) + return stmts + + def diff_options(self, newopts): + """Compare options lists and generate SQL SET or RESET clause + + :newopts: list of new options + :return: SQL SET / RESET clauses + + Generate ([SET|RESET storage_parameter=value) clauses from two + lists in the form of 'key=value' strings. + """ + def to_dict(optlist): + return dict(opt.split('=', 1) for opt in optlist) + + oldopts = {} + if self.options is not None: + oldopts = to_dict(self.options) + newopts = to_dict(newopts) + setclauses = [] + for key, val in list(newopts.items()): + if key not in oldopts: + setclauses.append("%s=%s" % (key, val)) + elif val != oldopts[key]: + setclauses.append("%s=%s" % (key, val)) + resetclauses = [] + for key, val in list(oldopts.items()): + if key not in newopts: + resetclauses.append("%s" % key) + clauses = '' + if setclauses: + clauses = "SET (%s)" % ', '.join(setclauses) + if resetclauses: + clauses += ', ' + if resetclauses: + clauses += "RESET (%s)" % ', '.join(resetclauses) + return clauses + + def alter(self, intable): + """Generate SQL to transform an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + + Compares the table to an input table and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if len(intable.columns) == 0: + raise KeyError("Table '%s' has no columns" % intable.name) + colnames = [col.name for col in self.columns if not col.dropped] + dbcols = len(colnames) + + colprivs = [] + base = "ALTER %s %s\n " % (self.objtype, self.qualname()) + # check input columns + for (num, incol) in enumerate(intable.columns): + if hasattr(incol, 'oldname'): + assert(self.columns[num].name == incol.oldname) + stmts.append(self.columns[num].rename(incol.name)) + # check existing columns + if num < dbcols and self.columns[num].name == incol.name: + (stmt, descr) = self.columns[num].alter(incol) + if stmt: + stmts.append(base + stmt) + colprivs.append(self.columns[num].diff_privileges(incol)) + if descr: + stmts.append(descr) + # add new columns + elif incol.name not in colnames and not incol.inherited: + (stmt, descr) = incol.add() + stmts.append(base + "ADD COLUMN %s" % stmt) + colprivs.append(incol.add_privs()) + if descr: + stmts.append(descr) + + newopts = [] + if intable.options is not None: + newopts = intable.options + diff_opts = self.diff_options(newopts) + if diff_opts: + stmts.append("ALTER %s %s %s" % (self.objtype, self.identifier(), + diff_opts)) + if colprivs: + stmts.append(colprivs) + if intable.tablespace is not None: + if self.tablespace is None \ + or self.tablespace != intable.tablespace: + stmts.append(base + "SET TABLESPACE %s" + % quote_id(intable.tablespace)) + elif self.tablespace is not None: + stmts.append(base + "SET TABLESPACE pg_default") + + stmts.append(super(Table, self).alter(intable)) + + return stmts + + def alter_drop_columns(self, intable): + """Generate SQL to drop columns from an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + + Compares the table to an input table and generates SQL + statements to drop any columns missing from the one + represented by the input. + """ + if len(intable.columns) == 0: + raise KeyError("Table '%s' has no columns" % intable.name) + stmts = [] + incolnames = set(attr.name for attr in intable.columns) + for attr in self.columns: + if attr.name not in incolnames: + if not getattr(attr, 'inherited', False): + stmts.append(attr.drop()) + + return stmts + + def data_export(self, dbconn, dirpath): + """Copy table data out to a file + + :param dbconn: database connection to use + :param dirpath: full path to the directory for the file to be created + """ + filepath = os.path.join(dirpath, self.extern_filename('data')) + if self.primary_key is not None: + order_by = [quote_id(self.columns[col - 1].name) + for col in self.primary_key.columns] + else: + order_by = ['%d' % (n + 1) for n in range(len(self.columns))] + dbconn.sql_copy_to( + "COPY (SELECT * FROM %s ORDER BY %s) TO STDOUT WITH CSV" % ( + self.qualname(), ', '.join(order_by)), filepath) + + def data_import(self, dirpath): + """Generate SQL to import data into a table + + :param dirpath: full path for the directory for the file + :return: list of SQL statements + """ + filepath = os.path.join(dirpath, self.extern_filename('data')) + stmts = [] + if hasattr(self, '_referred_by'): + for constr in self._referred_by: + stmts.append( + "ALTER TABLE %s DROP CONSTRAINT %s" + % (constr._table.qualname(), constr.name) + ) + stmts.append("TRUNCATE ONLY %s" % self.qualname()) + stmts.append(("\\copy ", self.qualname(), " from '", filepath, + "' csv")) + if hasattr(self, '_referred_by'): + for constr in self._referred_by: + stmts.append(constr.add()) + return stmts + + def get_implied_deps(self, db): + deps = super(Table, self).get_implied_deps(db) + for col in self.columns: + type = db.find_type(col.type) + if type is not None: + deps.add(type) + + # Check if the column depends on a sequence to avoid stating the + # dependency explicitly. + if col.default is not None: + m = re.match(r"nextval\('(.*)'::regclass\)", col.default) + if m: + seq = db.tables.find(m.group(1), self.schema) + if seq: + deps.add(seq) + if seq.owner_table is not None: + if not hasattr(self, '_owned_seqs'): + self._owned_seqs = [] + self._owned_seqs.append(seq) + + for pname in getattr(self, 'inherits', ()): + parent = db.tables.find(pname, self.schema) + assert parent is not None, "couldn't find parent table %s" % pname + deps.add(parent) + + return deps + + +OBJTYPES = ['table', 'sequence', 'view', 'materialized view'] + + +class ClassDict(DbObjectDict): + "The collection of tables and similar objects in a database" + + cls = DbClass + + def _from_catalog(self): + """Initialize the dictionary of tables by querying the catalogs""" + self.cls = Table + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + inhtbls = self.dbconn.fetchall(Table.inhquery()) + self.dbconn.rollback() + for tdata in inhtbls: + tbl = tdata["sub"] + partbl = tdata["parent"] + num = tdata["inhseqno"] + (sch, tbl) = split_schema_obj(tbl) + table = self[(sch, tbl)] + (sch, tbl) = split_schema_obj(partbl) + if table.schema == sch: + partbl = tbl + table.inherits.append(partbl) + self.cls = Sequence + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + obj.get_attrs(self.dbconn) + obj.get_dependent_table(self.dbconn) + from .view import View, MaterializedView + self.cls = View + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + self.cls = MaterializedView + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + from .view import View, MaterializedView + for k in inobjs: + inobj = inobjs[k] + objtype = None + for typ in OBJTYPES: + if k.startswith(typ): + objtype = typ + key = k[len(typ) + 1:] + if objtype is None: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'table': + self[(schema.name, key)] = table = Table.from_map( + key, schema, inobj) + if inobj and 'inherits' in inobj: + table.inherits = inobj.pop('inherits') + try: + newdb.columns.from_map(table, inobj['columns']) + except KeyError as exc: + if table.partition_by is not None: + exc.args = ("Table '%s' has no columns" % key, ) + raise + newdb.constraints.from_map(table, inobj) + if 'indexes' in inobj: + newdb.indexes.from_map(table, inobj['indexes']) + if 'rules' in inobj: + newdb.rules.from_map(table, inobj['rules']) + if 'triggers' in inobj: + newdb.triggers.from_map(table, inobj['triggers']) + elif objtype == 'sequence': + self[(schema.name, key)] = Sequence.from_map( + key, schema, inobj) + elif objtype == 'view': + self[(schema.name, key)] = view = View.from_map( + key, schema, inobj) + if 'triggers' in inobj: + newdb.triggers.from_map(view, inobj['triggers']) + elif objtype == 'materialized view': + self[(schema.name, key)] = mview = MaterializedView.from_map( + key, schema, inobj) + if 'indexes' in inobj: + newdb.indexes.from_map(mview, inobj['indexes']) + else: + raise KeyError("Unrecognized object type: %s" % k) + obj = self[(schema.name, key)] + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + + def find(self, obj, schema=None): + """Find a table given its name. + + The name can contain array type modifiers such as '[]' + + Return None if not found. + """ + sch, name = split_schema_obj(obj, schema) + name = name.rstrip('[]') + return self.get((sch, name)) + + def link_refs(self, dbcolumns, dbconstrs, dbindexes, dbrules, dbtriggers): + """Connect columns, constraints, etc. to their respective tables + + :param dbcolumns: dictionary of columns + :param dbconstrs: dictionary of constraints + :param dbindexes: dictionary of indexes + :param dbrules: dictionary of rules + :param dbtriggers: dictionary of triggers + + Links each list of table columns in `dbcolumns` to the + corresponding table. Fills the `foreign_keys`, + `unique_constraints`, `indexes` and `triggers` dictionaries + for each table by traversing the `dbconstrs`, `dbindexes` and + `dbtriggers` dictionaries, which are keyed by schema, table + and constraint, index or trigger name. + """ + seqs = [self[t] for t in self if isinstance(self[t], Sequence)] + for (sch, tbl) in dbcolumns: + if (sch, tbl) in self: + #assert isinstance(self[(sch, tbl)], Table) + self[(sch, tbl)].columns = dbcolumns[(sch, tbl)] + for col in dbcolumns[(sch, tbl)]: + col._table = self[(sch, tbl)] + if col.identity is not None: + for seq in seqs: + if col.table == seq.owner_table and \ + col.name == seq.owner_column: + col._owner_seq = seq + seq._owner_col = col + + # Normalize owner_column's to column names + for (sch, tbl) in self: + table = self[(sch, tbl)] + if isinstance(table, Sequence) and table.owner_table is not None: + if isinstance(table.owner_column, int): + table.owner_column = self[(sch, table.owner_table)]. \ + column_names()[table.owner_column - 1] + + for (sch, tbl, cns) in dbconstrs: + constr = dbconstrs[(sch, tbl, cns)] + if isinstance(constr, CheckConstraint) and constr.is_domain_check: + continue + assert self[(sch, tbl)] + constr._table = table = self[(sch, tbl)] + if isinstance(constr, CheckConstraint): + table.check_constraints.update({cns: constr}) + elif isinstance(constr, PrimaryKey): + table.primary_key = constr + elif isinstance(constr, ForeignKey): + # link referenced and referrer + constr._references = self[( + constr.ref_schema, constr.ref_table)] + self[ + (constr.ref_schema, constr.ref_table) + ]._referred_by.append(constr) + table.foreign_keys.update({cns: constr}) + elif isinstance(constr, UniqueConstraint): + table.unique_constraints.update({cns: constr}) + + def link_one(targdict, schema, tbl, objkey, objtype): + table = self[(schema, tbl)] + if not hasattr(table, objtype): + setattr(table, objtype, {}) + objdict = getattr(table, objtype) + objdict.update({objkey: targdict[(schema, tbl, objkey)]}) + + for (sch, tbl, idx) in dbindexes: + link_one(dbindexes, sch, tbl, idx, 'indexes') + for (sch, tbl, rul) in dbrules: + link_one(dbrules, sch, tbl, rul, 'rules') + dbrules[(sch, tbl, rul)]._table = self[(sch, tbl)] + for (sch, tbl, trg) in dbtriggers: + link_one(dbtriggers, sch, tbl, trg, 'triggers') + dbtriggers[(sch, tbl, trg)]._table = self[(sch, tbl)] diff --git a/pyrseas/dbobject/textsearch.py b/pyrseas/dbobject/textsearch.py new file mode 100644 index 0000000..0576b3a --- /dev/null +++ b/pyrseas/dbobject/textsearch.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.textsearch + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines eight classes: TSConfiguration, TSDictionary, + TSParser and TSTemplate derived from DbSchemaObject, and + TSConfigurationDict, TSDictionaryDict, TSParserDict and + TSTemplateDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_schema_obj + + +class TSConfiguration(DbSchemaObject): + """A text search configuration definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_config' + + def __init__(self, name, schema, description, owner, parser, + oid=None): + """Initialize the configuration + + :param name: configuration name (from cfgname) + :param description: comment text (from obj_description()) + :param schema: schema name (from cfgnamespace) + :param owner: owner name (from rolname via cfgowner) + :param parser: parser name (from prsname via cfgparser) + """ + super(TSConfiguration, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.parser = parser + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nc.nspname AS schema, cfgname AS name, + rolname AS owner, np.nspname || '.' || prsname AS parser, + obj_description(c.oid, 'pg_ts_config') AS description, c.oid + FROM pg_ts_config c JOIN pg_roles r ON (r.oid = cfgowner) + JOIN pg_ts_parser p ON (cfgparser = p.oid) + JOIN pg_namespace nc ON (cfgnamespace = nc.oid) + JOIN pg_namespace np ON (prsnamespace = np.oid) + WHERE nc.nspname != 'pg_catalog' + AND nc.nspname != 'information_schema' + ORDER BY nc.nspname, cfgname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a configuration instance from a YAML map + + :param name: configuration name + :param name: schema map + :param inobj: YAML map of the configuration + :return: configuration instance + """ + obj = TSConfiguration( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('parser', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH CONFIGURATION" + + def to_map(self, db, no_owner): + """Convert a text search configuration to a YAML-suitable format + + :return: dictionary + """ + dct = super(TSConfiguration, self).to_map(db, no_owner) + if '.' in self.parser: + (sch, pars) = self.parser.split('.') + if sch == self.schema: + dct['parser'] = pars + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the configuration + + :return: SQL statements + """ + clauses = [] + clauses.append("PARSER = %s" % self.parser) + return ["CREATE TEXT SEARCH CONFIGURATION %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + def get_implied_deps(self, db): + deps = super(TSConfiguration, self).get_implied_deps(db) + deps.add(db.tsparsers[split_schema_obj(self.parser, self.schema)]) + return deps + + +class TSConfigurationDict(DbObjectDict): + "The collection of text search configurations in a database" + + cls = TSConfiguration + + def from_map(self, schema, inconfigs): + """Initialize the dictionary of configs by examining the input map + + :param schema: schema owning the configurations + :param inconfigs: input YAML map defining the configurations + """ + for key in inconfigs: + if not key.startswith('text search configuration '): + raise KeyError("Unrecognized object type: %s" % key) + tsc = key[26:] + inobj = inconfigs[key] + self[(schema.name, tsc)] = TSConfiguration.from_map( + tsc, schema, inobj) + + +class TSDictionary(DbSchemaObject): + """A text search dictionary definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_dict' + + def __init__(self, name, schema, description, owner, template, options, + oid=None): + """Initialize the dictionary + + :param name: dictionary name (from dictname) + :param schema: schema name (from dictnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via dictowner) + :param template: template name (from dicttemplate) + :param options: initialization option string (from dictinitoption) + """ + super(TSDictionary, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.template = template + self.options = options + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, dictname AS name, rolname AS owner, + tmplname AS template, dictinitoption AS options, + obj_description(d.oid, 'pg_ts_dict') AS description, d.oid + FROM pg_ts_dict d JOIN pg_ts_template t ON (dicttemplate = t.oid) + JOIN pg_roles r ON (r.oid = dictowner) + JOIN pg_namespace n ON (dictnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, dictname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a dictionary instance from a YAML map + + :param name: dictionary name + :param name: schema map + :param inobj: YAML map of the dictionary + :return: dictionary instance + """ + obj = TSDictionary( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('template', None), + inobj.pop('options', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH DICTIONARY" + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the dictionary + + :return: SQL statements + """ + clauses = [] + clauses.append("TEMPLATE = %s" % self.template) + if self.options is not None: + clauses.append(self.options) + return ["CREATE TEXT SEARCH DICTIONARY %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSDictionaryDict(DbObjectDict): + "The collection of text search dictionaries in a database" + + cls = TSDictionary + + def from_map(self, schema, indicts): + """Initialize the dictionary of dictionaries by examining the input map + + :param schema: schema owning the dictionaries + :param indicts: input YAML map defining the dictionaries + """ + for key in indicts: + if not key.startswith('text search dictionary '): + raise KeyError("Unrecognized object type: %s" % key) + tsd = key[23:] + inobj = indicts[key] + self[(schema.name, tsd)] = TSDictionary.from_map( + tsd, schema, inobj) + + +class TSParser(DbSchemaObject): + """A text search parser definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_parser' + + def __init__(self, name, schema, description, start, gettoken, end, + headline, lextypes, + oid=None): + """Initialize the parser + + :param name: parser name (from prsname) + :param schema: schema name (from prsnamespace) + :param description: comment text (from obj_description()) + :param start: startup function (from prsstart) + :param gettoken: next-token function (from prstoken) + :param end: shutdown function (from prsend) + :param headline: headline function (from prsheadline) + :param lextypes: lextype function (from prslextype) + """ + super(TSParser, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.start = start + self.gettoken = gettoken + self.end = end + self.headline = headline + self.lextypes = lextypes + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, prsname AS name, + prsstart::regproc AS start, prstoken::regproc AS gettoken, + prsend::regproc AS end, prslextype::regproc AS lextypes, + prsheadline::regproc AS headline, + obj_description(p.oid, 'pg_ts_parser') AS description, p.oid + FROM pg_ts_parser p JOIN pg_namespace n ON (prsnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, prsname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a parser instance from a YAML map + + :param name: parser name + :param name: schema map + :param inobj: YAML map of the parser + :return: parser instance + """ + obj = TSParser(name, schema.name, inobj.pop('description', None), + inobj.pop('start', None), inobj.pop('gettoken', None), + inobj.pop('end', None), inobj.pop('headline', None), + inobj.pop('lextypes', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH PARSER" + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the parser + + :return: SQL statements + """ + clauses = [] + for attr in ['start', 'gettoken', 'end', 'lextypes']: + clauses.append("%s = %s" % (attr.upper(), getattr(self, attr))) + if self.headline is not None: + clauses.append("HEADLINE = %s" % self.headline) + return ["CREATE TEXT SEARCH PARSER %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSParserDict(DbObjectDict): + "The collection of text search parsers in a database" + + cls = TSParser + + def from_map(self, schema, inparsers): + """Initialize the dictionary of parsers by examining the input map + + :param schema: schema owning the parsers + :param inparsers: input YAML map defining the parsers + """ + for key in inparsers: + if not key.startswith('text search parser '): + raise KeyError("Unrecognized object type: %s" % key) + tsp = key[19:] + inobj = inparsers[key] + self[(schema.name, tsp)] = TSParser.from_map(tsp, schema, inobj) + + +class TSTemplate(DbSchemaObject): + """A text search template definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_template' + + def __init__(self, name, schema, description, init, lexize, + oid=None): + """Initialize the template + + :param name: template name (from tmplname) + :param schema: schema name (from tmplnamespace) + :param description: comment text (from obj_description()) + :param init: initialization function (from tmplinit) + :param lexize: lexize function (from tmpllexize) + """ + super(TSTemplate, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.init = init + self.lexize = lexize + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, tmplname AS name, p.oid, + tmplinit::regproc AS init, tmpllexize::regproc AS lexize, + obj_description(p.oid, 'pg_ts_template') AS description + FROM pg_ts_template p + JOIN pg_namespace n ON (tmplnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, tmplname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a template instance from a YAML map + + :param name: template name + :param name: schema map + :param inobj: YAML map of the template + :return: template instance + """ + obj = TSTemplate(name, schema.name, inobj.pop('description', None), + inobj.pop('init', None), inobj.pop('lexize', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH TEMPLATE" + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the template + + :return: SQL statements + """ + clauses = [] + if self.init is not None: + clauses.append("INIT = %s" % self.init) + clauses.append("LEXIZE = %s" % self.lexize) + return ["CREATE TEXT SEARCH TEMPLATE %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSTemplateDict(DbObjectDict): + "The collection of text search templates in a database" + + cls = TSTemplate + + def from_map(self, schema, intemplates): + """Initialize the dictionary of templates by examining the input map + + :param schema: schema owning the templates + :param intemplates: input YAML map defining the templates + """ + for key in intemplates: + if not key.startswith('text search template '): + raise KeyError("Unrecognized object type: %s" % key) + tst = key[21:] + inobj = intemplates[key] + self[(schema.name, tst)] = TSTemplate.from_map(tst, schema, inobj) diff --git a/pyrseas/dbobject/trigger.py b/pyrseas/dbobject/trigger.py new file mode 100644 index 0000000..0a28557 --- /dev/null +++ b/pyrseas/dbobject/trigger.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.trigger + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Trigger derived from + DbSchemaObject, and TriggerDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, split_schema_obj +from .function import split_schema_func, join_schema_func + +EVENT_TYPES = ['insert', 'delete', 'update', 'truncate'] + + +class Trigger(DbSchemaObject): + """A procedural language trigger""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_trigger' + + def __init__(self, name, schema, table, description, procedure, timing, + level, events, constraint=False, deferrable=False, + initially_deferred=False, referencing_new=None, + referencing_old=None, columns=[], condition=None, + arguments='', + oid=None): + """Initialize the trigger + + :param name: trigger name (from tgname) + :param schema: schema name (from tgnamespace) + :param table: table name (from relname via tgrelid) + :param description: comment text (from obj_description()) + :param procedure: function to call (from tgfoid) + :param level: row/statement (from tgtype bit 0) + :param timing: before/after/instead of (from tgtype bit 1 and 6) + :param events: insert/update/delete/truncate (from tgtype bits 2-5) + :param constraint: is it a constraint trigger? (from contype) + :param deferrable: is it deferrable? (from tgdeferrrable) + :param initially_deferred: initially deferred? (from tginitdeferred) + :param referencing_new: NEW transition relation name + :param referencing_old: OLD transition relation name + :param columns: array of column numbers (from tgattr) + :param condition: WHEN condition + :param arguments: arguments to pass to trigger (from tgargs) + """ + super(Trigger, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = table + if procedure[-2:] == '()': + procedure = procedure[:-2] + if '.' in procedure: + self.procedure = split_schema_obj(procedure, self.schema) + else: + self.procedure = procedure + if arguments and '\\000' in arguments: + self.arguments = ", ".join(["'%s'" % a for a in + arguments.split('\\000')[:-1]]) + else: + self.arguments = arguments if len(arguments) > 0 else None + # see Postgres include/catalog/pg_trigger.h + if isinstance(timing, int): + if timing == (1 << 1): + self.timing = 'before' + elif timing == (1 << 6): + self.timing = 'instead of' + else: + self.timing = 'after' + else: + self.timing = timing + self.level = level + if isinstance(events, int): + self.events = [] + for (n, ev) in enumerate(EVENT_TYPES): + if events & 1 << n: + self.events.append(ev) + else: + assert isinstance(events, list), "Events must be a list" + self.events = events + self.constraint = constraint + self.deferrable = deferrable + self.initially_deferred = initially_deferred + self.referencing_new = referencing_new + self.referencing_old = referencing_old + self.columns = columns + self.condition = condition + if condition is not None and condition.startswith('CREATE '): + if 'WHEN (' in condition: + self.condition = condition[condition.index("WHEN (")+6: + condition.index(") EXECUTE ")] + else: + self.condition = None + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS table, tgname AS name, + tgfoid::regprocedure AS procedure, + CASE WHEN tgtype::integer::bit = '1' THEN 'row' + ELSE 'statement' END AS level, + (tgtype::integer::bit(7) & B'1000010')::integer AS timing, + (tgtype >> 2)::integer::bit(4)::integer AS events, + CASE WHEN contype = 't' THEN true ELSE false END AS + constraint, + tgdeferrable AS deferrable, + tginitdeferred AS initially_deferred, tgattr AS columns, + encode(tgargs, 'escape') AS arguments, + pg_get_triggerdef(t.oid) AS condition, + obj_description(t.oid, 'pg_trigger') AS description, + tgnewtable AS referencing_new, + tgoldtable AS referencing_old, + t.oid + FROM pg_trigger t JOIN pg_class c ON (t.tgrelid = c.oid) + JOIN pg_namespace n ON (c.relnamespace = n.oid) + JOIN pg_roles ON (n.nspowner = pg_roles.oid) + LEFT JOIN pg_constraint cn ON (tgconstraint = cn.oid) + WHERE NOT tgisinternal + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND t.tgfoid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a trigger instance from a YAML map + + :param name: trigger name + :param table: table map + :param inobj: YAML map of the trigger + :return: trigger instance + """ + proc = inobj.pop("procedure") + args = "" + if isinstance(proc, str): + if proc[-2:] == "()": + proc = proc[:-2] + elif proc[-1:] == ')': + proc, args = proc[:-1].split('(') + else: # should be a dict + args = proc.pop("arguments", None) + proc = proc.pop("name") + obj = Trigger( + name, table.schema, table.name, inobj.pop('description', None), + proc, inobj.pop('timing', None), inobj.pop('level', 'statement'), + inobj.pop('events', []), inobj.pop('constraint', False), + inobj.pop('deferrable', False), + inobj.pop('initially_deferred', False), + inobj.pop('referencing_new', None), + inobj.pop('referencing_old', None), + inobj.pop('columns', []), inobj.pop('condition', None), args) + obj.set_oldname(inobj) + return obj + + def identifier(self): + """Returns a full identifier for the trigger + + :return: string + """ + return "%s ON %s" % (quote_id(self.name), self._table.qualname()) + + def to_map(self, db): + """Convert a trigger to a YAML-suitable format + + :return: dictionary + """ + dct = super(Trigger, self).to_map(db) + schfunc = join_schema_func(self.procedure) + if self.arguments is not None: + dct["procedure"] = {"name": schfunc, "arguments": self.arguments} + else: + dct["procedure"] = schfunc + dct.pop("arguments") + for attr in ['constraint', 'deferrable', 'initially_deferred']: + if dct[attr] is False: + del dct[attr] + if len(self.columns) > 0: + dct['columns'] = [self._table.column_names()[int(k) - 1] + for k in self.columns.split()] + else: + del dct['columns'] + if self.condition is None: + del dct['condition'] + if self.referencing_new is None: + del dct['referencing_new'] + if self.referencing_old is None: + del dct['referencing_old'] + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the trigger + + :return: SQL statements + """ + constr = defer = '' + if self.constraint: + constr = "CONSTRAINT " + if self.deferrable: + defer = "DEFERRABLE " + if self.initially_deferred: + defer += "INITIALLY DEFERRED" + if defer: + defer = '\n ' + defer + evts = " OR ".join(self.events).upper() + if len(self.columns) > 0 and 'update' in self.events: + evts = evts.replace("UPDATE", "UPDATE OF %s" % ( + ", ".join(self.columns))) + cond = '' + if self.condition is not None: + cond = "\n WHEN (%s)" % self.condition + if isinstance(self.procedure, tuple): + procname = "%s.%s" % self.procedure + else: + procname = self.procedure + if self.arguments is None: + args = "" + else: + args = self.arguments + referencing = '' + if self.referencing_new or self.referencing_old: + referencing = "\n REFERENCING" + if self.referencing_new: + referencing += " NEW TABLE AS %s" % self.referencing_new + if self.referencing_old: + referencing += " OLD TABLE AS %s" % self.referencing_old + return ["CREATE %sTRIGGER %s\n %s %s ON %s%s%s\n FOR EACH %s" + "%s\n EXECUTE PROCEDURE %s(%s)" % ( + constr, quote_id(self.name), self.timing.upper(), evts, + self._table.qualname(), defer, referencing, + self.level.upper(), cond, procname, args)] + + def alter(self, inobj): + """Generate SQL to transform an existing trigger + + :param inobj: a YAML map defining the new trigger + :return: list of SQL statements + """ + stmts = [] + if self.procedure != inobj.procedure or \ + self.arguments != inobj.arguments or self.events != inobj.events \ + or self.level != inobj.level or self.timing != inobj.timing \ + or self.referencing_new != inobj.referencing_new \ + or self.referencing_old != inobj.referencing_old: + stmts.append(self.drop()) + stmts.append(inobj.create()) + stmts.append(self.diff_description(inobj)) + return stmts + + def get_implied_deps(self, db): + deps = super(Trigger, self).get_implied_deps(db) + + deps.add(db.tables[self.schema, self.table]) + + # short-circuit augment triggers + if hasattr(self, '_iscfg'): + return deps + + # the trigger procedure can have arguments, but the trigger definition + # has always none (they are accessed through `tg_argv`). + if isinstance(self.procedure, tuple): + fschema, fname = self.procedure + deps.add(db.functions[fschema, fname, '']) + + return deps + + +class TriggerDict(DbObjectDict): + "The collection of triggers in a database" + cls = Trigger + + def from_map(self, table, intriggers): + """Initialize the dictionary of triggers by converting the input map + + :param table: table owning the triggers + :param intriggers: YAML map defining the triggers + """ + for trg in intriggers: + inobj = intriggers[trg] + self[(table.schema, table.name, trg)] = Trigger.from_map( + trg, table, inobj) diff --git a/pyrseas/dbobject/view.py b/pyrseas/dbobject/view.py new file mode 100644 index 0000000..2160601 --- /dev/null +++ b/pyrseas/dbobject/view.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.view + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: View derived from DbClass and + MaterializedView derived from View. +""" +from pyrseas.yamlutil import MultiLineStr +from . import commentable, ownable, grantable +from .table import DbClass +from .column import Column + + +class View(DbClass): + """A database view definition + + A view is identified by its schema name and view name. + """ + def __init__(self, name, schema, description, owner, privileges, + definition, + oid=None): + """Initialize the view + + :param name-privileges: see DbClass.__init__ params + :param definition: prettified definition (from pg_getviewdef) + """ + super(View, self).__init__(name, schema, description, owner, + privileges) + self.definition = MultiLineStr(definition) + self.triggers = {} + self.columns = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_viewdef(c.oid, TRUE) AS definition, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'v' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a view instance from a YAML map + + :param name: view name + :param name: schema map + :param inobj: YAML map of the view + :return: view instance + """ + obj = View( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('definition', None)) + if "columns" in inobj: + obj.columns = [Column(list(col.keys())[0], schema.name, name, + i + 1, + list(col.values())[0].get("type", None)) + for i, col in enumerate(inobj.get("columns"))] + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + privobjtype = "TABLE" + + @property + def allprivs(self): + return 'arwdDxt' + + def to_map(self, db, opts): + """Convert a view to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return None + dct = super(View, self).to_map(db, opts.no_owner, opts.no_privs) + dct['columns'] = [col.to_map(db, opts.no_privs) + for col in self.columns] + if 'dependent_funcs' in dct: + dct.pop('dependent_funcs') + if len(self.triggers) > 0: + for key in list(self.triggers.values()): + dct['triggers'].update(self.triggers[key.name].to_map(db)) + else: + dct.pop('triggers') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newdefn=None): + """Return SQL statements to CREATE the view + + :return: SQL statements + """ + defn = newdefn or self.definition + if defn[-1:] == ';': + defn = defn[:-1] + return ["CREATE%s VIEW %s AS\n %s" % ( + newdefn and " OR REPLACE" or '', self.qualname(), defn)] + + def alter(self, inview, dbversion=None): + """Generate SQL to transform an existing view + + :param inview: a YAML map defining the new view + :return: list of SQL statements + + Compares the view to an input view and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + for col in self.columns: + if col.name != inview.columns[col.number - 1].name: + raise KeyError("Cannot change name of view column '%s'" + % col.name) + if col.type != inview.columns[col.number - 1].type: + raise TypeError("Cannot change datatype of view column '%s'" + % col.name) + if self.definition != inview.definition: + stmts.append(self.create(dbversion, inview.definition)) + stmts.append(super(View, self).alter(inview)) + return stmts + + +class MaterializedView(View): + """A materialized view definition + + A materialized view is identified by its schema name and view name. + """ + def __init__(self, name, schema, description, owner, privileges, + definition, with_data=False, + oid=None): + """Initialize the materialized view + + :param name-privileges: see DbClass.__init__ params + :param definition: prettified definition (from pg_getviewdef) + :param with_data: is view populated (from relispopulated) + """ + super(MaterializedView, self).__init__( + name, schema, description, owner, privileges, definition) + self.with_data = with_data + self.indexes = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_viewdef(c.oid, TRUE) AS definition, + relispopulated AS with_data, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'm' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a materialized view instance from a YAML map + + :param name: view name + :param name: schema map + :param inobj: YAML map of the view + :return: materialized view instance + """ + obj = MaterializedView( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('definition', None)) + if "columns" in inobj: + obj.columns = [Column(list(col.keys())[0], schema.name, name, + i + 1, + list(col.values())[0].get("type", None)) + for i, col in enumerate(inobj.get("columns"))] + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "MATERIALIZED VIEW" + + def to_map(self, db, opts): + """Convert a materialized view to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return None + mvw = super(MaterializedView, self).to_map(db, opts) + if len(self.indexes) > 0: + for k in list(self.indexes.values()): + mvw['indexes'].update(self.indexes[k.name].to_map(db)) + else: + mvw.pop('indexes') + return mvw + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newdefn=None): + """Return SQL statements to CREATE the materialized view + + :return: SQL statements + """ + defn = newdefn or self.definition + if defn[-1:] == ';': + defn = defn[:-1] + return ["CREATE %s %s AS\n %s" % ( + self.objtype, self.qualname(), defn)] diff --git a/pyrseas/dbtoyaml.py b/pyrseas/dbtoyaml.py new file mode 100644 index 0000000..5f50b32 --- /dev/null +++ b/pyrseas/dbtoyaml.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""dbtoyaml - extract the schema of a PostgreSQL database in YAML format""" + +from __future__ import print_function +import sys + +from pyrseas import __version__ +from pyrseas.yamlutil import yamldump +from pyrseas.database import Database +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(schema=None): + """Convert database table specifications to YAML.""" + parser = cmd_parser("Extract the schema of a PostgreSQL database in " + "YAML format", __version__) + parser.add_argument('-m', '--multiple-files', action='store_true', + help='output to multiple files (metadata directory)') + parser.add_argument('-O', '--no-owner', action='store_true', + help='exclude object ownership information') + parser.add_argument('-x', '--no-privileges', action='store_true', + dest='no_privs', + help='exclude privilege (GRANT/REVOKE) information') + group = parser.add_argument_group("Object inclusion/exclusion options", + "(each can be given multiple times)") + group.add_argument('-n', '--schema', metavar='SCHEMA', dest='schemas', + action='append', default=[], + help="extract the named schema(s) (default all)") + group.add_argument('-N', '--exclude-schema', metavar='SCHEMA', + dest='excl_schemas', action='append', default=[], + help="do NOT extract the named schema(s) " + "(default none)") + group.add_argument('-t', '--table', metavar='TABLE', dest='tables', + action='append', default=[], + help="extract the named table(s) (default all)") + group.add_argument('-T', '--exclude-table', metavar='TABLE', + dest='excl_tables', action='append', default=[], + help="do NOT extract the named table(s) " + "(default none)") + parser.set_defaults(schema=schema) + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + if options.multiple_files and output: + parser.error("Cannot specify both --multiple-files and --output") + + db = Database(cfg) + dbmap = db.to_map() + + if not options.multiple_files: + print(yamldump(dbmap), file=output or sys.stdout) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/lib/__init__.py b/pyrseas/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyrseas/lib/dbconn.py b/pyrseas/lib/dbconn.py new file mode 100644 index 0000000..ce0776c --- /dev/null +++ b/pyrseas/lib/dbconn.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +""" + lib.dbconn + ~~~~~~~~~~ + + A `DbConnection` is a helper class representing a connection to a + PostgreSQL database. +""" +import sys + +from psycopg import connect +from psycopg.extras import dict_row + + +class DbConnection(object): + """A database connection, possibly disconnected""" + + def __init__(self, dbname, user=None, pswd=None, host=None, port=None): + """Initialize the connection information + + :param dbname: database name + :param user: user name + :param pswd: user password + :param host: host name + :param port: host port number + """ + self.dbname = dbname + self.user = '' if user is None else " user=%s" % user + self.pswd = '' if pswd is None else " password=%s" % pswd + self.host = '' if host is None else "host=%s " % host + self.port = '' if port is None else "port=%d " % port + self.conn = None + + def connect(self): + """Connect to the database""" + try: + self.conn = connect("%s%sdbname=%s%s%s" % ( + self.host, self.port, self.dbname, self.user, self.pswd)) + except Exception as exc: + if str(exc)[:6] == 'FATAL:': + sys.exit("Database connection error: %s" % str(exc)[8:]) + else: + raise exc + + def close(self): + """Close the database connection""" + if self.conn and not self.conn.closed: + self.conn.close() + self.conn = None + + def commit(self): + """Commit currently open transaction""" + self.conn.commit() + + def rollback(self): + """Roll back currently open transaction""" + self.conn.rollback() + + def execute(self, query, args=None): + """Create a cursor, execute a query and return the cursor + + :param query: text of the statement to execute + :param args: arguments to query + :return: cursor + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + try: + curs.execute(query, args) + except Exception as exc: + self.conn.rollback() + curs.close() + raise exc + return curs + + def fetchone(self, query, args=None): + """Execute a single row SELECT query and return row + + :param query: a SELECT query to be executed + :param args: arguments to query + :return: a psycopg DictRow + + The cursor is closed. + """ + curs = self.execute(query, args) + row = curs.fetchone() + curs.close() + return row + + def fetchall(self, query, args=None): + """Execute a SELECT query and return rows + + :param query: a SELECT query to be executed + :param args: arguments to query + :return: a list of psycopg DictRow's + + The cursor is closed. + """ + curs = self.execute(query, args) + rows = curs.fetchall() + curs.close() + return rows + + def sql_copy_to(self, sql, path): + """Execute an SQL COPY command to a file + + :param sql: SQL copy command + :param path: file name/path to copy into + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + with curs.copy(sql) as copy: + with open(path, "wb") as f: + for data in copy: + f.write(bytes(data)) + + def copy_from(self, path, table): + """Execute a COPY command from a file in CSV format + + :param path: file name/path to copy from + :param table: possibly schema qualified table name + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + with open(path, 'r') as f: + try: + with curs.copy("COPY %s FROM STDIN WITH CSV" % table) as copy: + while data := f.read(): + copy.write(data) + except: + raise + curs.close() diff --git a/pyrseas/lib/dbutils.py b/pyrseas/lib/dbutils.py new file mode 100644 index 0000000..e553ce0 --- /dev/null +++ b/pyrseas/lib/dbutils.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +"""Database utility functions and classes + +These are primarily to assist in testing Pyrseas, i.e., without having +to depend on the application-level DbConnection. +""" +import os + +from psycopg import connect +from psycopg.rows import dict_row + + +def pgconnect(dbname, user=None, host=None, port=None, autocommit=False): + "Connect to a Postgres database using psycopg" + user = '' if user is None else " user=%s" % user + host = '' if host is None else "host=%s " % host + port = '' if port is None else "port=%d " % port + return connect("%s%sdbname=%s%s" % (host, port, dbname, user), + autocommit=autocommit) + + +def pgexecute(dbconn, oper, args=None): + "Execute an operation using a cursor" + curs = dbconn.cursor() + try: + curs.execute(oper, args) + except: + curs.close() + dbconn.rollback() + raise + return curs + + +ADMIN_DB = os.environ.get("PG_ADMIN_DB", 'postgres') +CREATE_DDL = "CREATE DATABASE %s TEMPLATE = template0" + + +class PostgresDb(object): + """A PostgreSQL database connection + + This is separate from the one used by DbConnection, because tests + need to create and drop databases and other objects, + independently. + """ + def __init__(self, name, user, host, port): + self.name = name + self.conn = None + self.user = user + self.host = host + self.port = port and int(port) + self._version = 0 + + def connect(self): + """Connect to the database + + If we're not already connected we first connect to the admin + database and see if the given database exists. If it doesn't, + we create and then connect to it. + """ + if not self.conn: + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port) + curs = pgexecute(conn, + "SELECT 1 FROM pg_database WHERE datname = '%s'" % + self.name) + row = curs.fetchone() + if not row: + curs.close() + conn2 = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn2, CREATE_DDL % self.name) + curs.close() + conn.close() + self.conn = pgconnect(self.name, self.user, self.host, self.port) + curs = pgexecute(self.conn, "SHOW server_version_num") + vers = curs.fetchone() + self._version = int(vers["server_version_num"]) + + def close(self): + "Close the connection if still open" + if not self.conn: + return ValueError + self.conn.close() + + @property + def version(self): + return self._version + + def create(self): + "Drop the database if it exists and re-create it" + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn, "DROP DATABASE IF EXISTS %s" % self.name) + curs = pgexecute(conn, CREATE_DDL % self.name) + curs.close() + conn.close() + + def drop(self): + "Drop the database" + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn, "DROP DATABASE %s" % self.name) + curs.close() + conn.close() + + def execute(self, stmt, args=None): + "Execute a DDL statement" + curs = pgexecute(self.conn, stmt, args) + curs.close() + + def execute_commit(self, stmt, args=None): + "Execute a DDL statement and commit" + self.execute(stmt, args) + self.conn.commit() + + def fetchone(self, query, args=None): + "Execute a query and return one row" + try: + curs = pgexecute(self.conn, query, args) + except Exception as exc: + raise exc + row = curs.fetchone() + curs.close() + return row diff --git a/pyrseas/testutils.py b/pyrseas/testutils.py new file mode 100644 index 0000000..bbcee1a --- /dev/null +++ b/pyrseas/testutils.py @@ -0,0 +1,412 @@ +# -*- coding: utf-8 -*- +"""Utility functions and classes for testing Pyrseas""" + +import sys +import os +import getpass +import tempfile +import glob +import subprocess +from unittest import TestCase + +import yaml + +from pyrseas.config import Config +from pyrseas.database import Database +from pyrseas.augmentdb import AugmentDatabase +from pyrseas.lib.dbutils import pgexecute, PostgresDb + + +def fix_indent(stmt): + "Fix specifications which are in a new line with indentation" + return stmt.replace(' ', ' ').replace(' ', ' ').replace('\n ', ' '). \ + replace('( ', '(') + + +def remove_temp_files(tmpdir, prefix=''): + "Remove files in a temporary directory" + for tfile in glob.glob(os.path.join(tmpdir, prefix + '*')): + if os.path.isdir(tfile): + for entry in os.listdir(tfile): + entry = os.path.join(tmpdir, tfile, entry) + if os.path.isdir(entry): + for file in os.listdir(entry): + os.remove(os.path.join(entry, file)) + os.rmdir(entry) + else: + os.remove(entry) + os.rmdir(tfile) + else: + os.remove(tfile) + + +TEST_DBNAME = os.environ.get("PYRSEAS_TEST_DB", 'pyrseas_testdb') +TEST_USER = os.environ.get("PYRSEAS_TEST_USER", getpass.getuser()) +TEST_HOST = os.environ.get("PYRSEAS_TEST_HOST", None) +TEST_PORT = int(os.environ.get("PYRSEAS_TEST_PORT", 5432)) +PG_OWNER = 'postgres' +TEST_DIR = os.path.join(tempfile.gettempdir(), + os.environ.get("PYRSEAS_TEST_DIR", 'pyrseas_test')) +TRAVIS = (os.environ.get("TRAVIS", 'false') == 'true') + + +class PgTestDb(PostgresDb): + """A PostgreSQL database connection for testing.""" + + def clear(self): + "Drop schemas and other objects" + STD_DROP = 'DROP %s IF EXISTS "%s" CASCADE' + # Schemas + curs = pgexecute( + self.conn, + """SELECT nspname FROM pg_namespace + WHERE nspname != 'information_schema' + AND substring(nspname for 3) != 'pg_' + ORDER BY nspname""") + objs = curs.fetchall() + curs.close() + self.conn.rollback() + for obj in objs: + self.execute(STD_DROP % ('SCHEMA', obj["nspname"])) + self.conn.commit() + + # Extensions + curs = pgexecute( + self.conn, + """SELECT extname FROM pg_extension + JOIN pg_namespace n ON (extnamespace = n.oid) + WHERE nspname != 'information_schema' + AND extname != 'plpgsql'""") + exts = curs.fetchall() + curs.close() + self.conn.rollback() + for ext in exts: + self.execute(STD_DROP % ('EXTENSION', ext["extname"])) + self.conn.commit() + + # User mappings + curs = pgexecute( + self.conn, + """SELECT CASE umuser WHEN 0 THEN 'PUBLIC' ELSE + pg_get_userbyid(umuser) END AS username, s.srvname + FROM pg_user_mappings u + JOIN pg_foreign_server s ON (srvid = s.oid)""") + umaps = curs.fetchall() + curs.close() + self.conn.rollback() + for ump in umaps: + self.execute('DROP USER MAPPING IF EXISTS FOR "%s" SERVER "%s"' % ( + ump["username"], ump["srvname"])) + self.conn.commit() + + # Servers + curs = pgexecute(self.conn, "SELECT srvname FROM pg_foreign_server") + servs = curs.fetchall() + curs.close() + self.conn.rollback() + for srv in servs: + self.execute(STD_DROP % ('SERVER', srv["srvname"])) + self.conn.commit() + + # Foreign data wrappers + curs = pgexecute(self.conn, + "SELECT fdwname FROM pg_foreign_data_wrapper") + fdws = curs.fetchall() + curs.close() + self.conn.rollback() + for fdw in fdws: + self.execute(STD_DROP % ('FOREIGN DATA WRAPPER', fdw["fdwname"])) + self.conn.commit() + + # Create default schema + self.execute("CREATE SCHEMA sd") + self.execute("set search_path='sd', 'pg_catalog'") + self.conn.commit() + + def is_plpgsql_installed(self): + "Is PL/pgSQL installed?" + curs = pgexecute(self.conn, + "SELECT 1 FROM pg_language WHERE lanname = 'plpgsql'") + row = curs.fetchone() + curs.close() + return row and True + + def is_superuser(self): + "Is current user a superuser?" + curs = pgexecute(self.conn, "SELECT 1 FROM pg_roles WHERE rolsuper " + "AND rolname = CURRENT_USER ") + row = curs.fetchone() + curs.close() + return row and True + + +def _connect_clear(dbname): + db = PgTestDb(dbname, TEST_USER, TEST_HOST, TEST_PORT) + db.connect() + db.clear() + return db + + +class PyrseasTestCase(TestCase): + """Base class for most test cases""" + + def setUp(self): + self.db = _connect_clear(TEST_DBNAME) + self.cfg = Config(sys_only=True) + if 'database' not in self.cfg: + self.cfg.update(database={}) + dbc = self.cfg['database'] + dbc['dbname'] = self.db.name + dbc['username'] = self.db.user + dbc['password'] = None + dbc['host'] = self.db.host + dbc['port'] = self.db.port + + def tearDown(self): + self.db.close() + + def database(self): + """The Pyrseas Database instance""" + return Database(self.cfg) + + def config_options(self, **kwargs): + class Opts(): + def __init__(self, **kwargs): + [setattr(self, opt, val) for opt, val in list(kwargs.items())] + self.cfg['options'] = Opts(**kwargs) + + +class DatabaseToMapTestCase(PyrseasTestCase): + """Base class for "database to map" test cases""" + + superuser = False + + def to_map(self, stmts, config={}, schemas=[], tables=[], no_owner=True, + no_privs=True, superuser=False, multiple_files=False): + """Execute statements and return a database map. + + :param stmts: list of SQL statements to execute + :param config: dictionary of configuration information + :param schemas: list of schemas to map + :param tables: list of tables to map + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :param superuser: must be superuser to run + :param multiple_files: emulate --multiple_files option + :return: possibly trimmed map of database + """ + if (self.superuser or superuser) and not self.db.is_superuser(): + self.skipTest("Must be a superuser to run this test") + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + if multiple_files: + self.cfg.merge({'files': {'metadata_path': os.path.join( + TEST_DIR, self.cfg['repository']['metadata'])}}) + if 'datacopy' in config: + self.cfg.merge({'files': {'data_path': os.path.join( + TEST_DIR, self.cfg['repository']['data'])}}) + self.config_options(schemas=schemas, tables=tables, no_owner=no_owner, + no_privs=no_privs, multiple_files=multiple_files) + self.cfg.merge(config) + return self.database().to_map() + + def yaml_load(self, filename, subdir=None): + """Read a file in the metadata_path and process it with YAML load + + :param filename: name of the file + :param subdir: name of a subdirectory where the file is located + :return: YAML dictionary + """ + with open(os.path.join(self.cfg['files']['metadata_path'], + subdir or '', filename), 'r') as f: + inmap = f.read() + return yaml.safe_load(inmap) + + def remove_tempfiles(self): + remove_temp_files(TEST_DIR) + + @staticmethod + def sort_privileges(data): + try: + sorted_privlist = [] + for sortedItem in sorted([list(i.keys())[0] + for i in data['privileges']]): + sorted_privlist.append( + [item for item in data['privileges'] + if list(item.keys())[0] == sortedItem][0]) + data['privileges'] = sorted_privlist + finally: + return data + + +class InputMapToSqlTestCase(PyrseasTestCase): + """Base class for "input map to SQL" test cases""" + + superuser = False + + def to_sql(self, inmap, stmts=None, config={}, superuser=False, schemas=[], + revert=False, quote_reserved=False): + """Execute statements and compare database to input map. + + :param inmap: dictionary defining target database + :param stmts: list of SQL database setup statements + :param config: dictionary of configuration information + :param superuser: indicates test requires superuser privilege + :param schemas: list of schemas to diff + :param revert: generate statements to back out changes + :param quote_reserved: fetch reserved words + :return: list of SQL statements + """ + if (self.superuser or superuser) and not self.db.is_superuser(): + self.skipTest("Must be a superuser to run this test") + if stmts: + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + + if 'datacopy' in config: + self.cfg.merge({'files': {'data_path': os.path.join( + TEST_DIR, self.cfg['repository']['data'])}}) + self.config_options(schemas=schemas, revert=revert), + self.cfg.merge(config) + return self.database().diff_map(inmap, quote_reserved=quote_reserved) + + def std_map(self, plpgsql_installed=False): + "Return a standard schema map for the default database" + base = {'schema sd': { + 'owner': self.db.user, + 'privileges': []}} + base.update({'extension plpgsql': { + 'schema': 'pg_catalog', 'owner': PG_OWNER, + 'description': "PL/pgSQL procedural language"}}) + return base + + +TEST_DBNAME_SRC = os.environ.get("PYRSEAS_TEST_DB_SRC", 'pyrseas_testdb_src') + + +class DbMigrateTestCase(TestCase): + + @classmethod + def setUpClass(cls): + cls.srcdb = _connect_clear(TEST_DBNAME_SRC) + cls.db = _connect_clear(TEST_DBNAME) + progdir = os.path.abspath(os.path.dirname(__file__)) + cls.dbtoyaml = os.path.join(progdir, 'dbtoyaml.py') + cls.yamltodb = os.path.join(progdir, 'yamltodb.py') + cls.tmpdir = TEST_DIR + if not os.path.exists(cls.tmpdir): + os.mkdir(cls.tmpdir) + + def add_public_schema(self, db): + db.execute("CREATE SCHEMA IF NOT EXISTS public") + db.execute("ALTER SCHEMA public OWNER TO postgres") + db.execute("COMMENT ON SCHEMA public IS " + "'standard public schema'") + db.execute("DROP SCHEMA IF EXISTS sd") + db.conn.commit() + + def remove_public_schema(self, db): + db.execute("DROP SCHEMA IF EXISTS public CASCADE") + db.conn.commit() + + @classmethod + def remove_tempfiles(cls, prefix): + remove_temp_files(cls.tmpdir, prefix) + + def execute_script(self, path, scriptname): + scriptfile = os.path.join(os.path.abspath(os.path.dirname(path)), + scriptname) + lines = [] + with open(scriptfile, 'r') as fd: + lines = [line.strip() for line in fd if line != '\n' and + not line.startswith('--')] + self.srcdb.execute_commit(' '.join(lines)) + + def tempfile_path(self, filename): + return os.path.join(self.tmpdir, filename) + + def _db_params(self): + args = [] + if self.db.host is not None: + args.append("--host=%s" % self.db.host) + if self.db.port is not None: + args.append("--port=%d " % self.db.port) + if self.db.user is not None: + args.append("--username=%s" % self.db.user) + return args + + def lines(self, the_file): + with open(the_file) as f: + lines = f.readlines() + return lines + + def run_pg_dump(self, dumpfile, srcdb=False, incldata=False): + """Run pg_dump using special scripts or directly (on Travis-CI) + + :param dumpfile: path to the pg_dump output file + :param srcdb: run against source database + """ + if TRAVIS: + pg_dumpver = 'pg_dump' + else: + v = self.srcdb._version + pg_dumpver = "pg_dump%d%d" % (v // 10000, + (v - v // 10000 * 10000) // 100) + if sys.platform == 'win32': + pg_dumpver += '.bat' + dbname = self.srcdb.name if srcdb else self.db.name + args = [pg_dumpver] + args.extend(self._db_params()) + if not incldata: + args.extend(['-s']) + args.extend(['-f', dumpfile, dbname]) + subprocess.check_call(args) + + def invoke(self, args): + args.insert(0, sys.executable) + path = [os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))] + path.append(os.path.abspath(os.path.join(os.path.dirname( + yaml.__file__), '..'))) + env = os.environ.copy() + env.update({'PYTHONPATH': os.pathsep.join(path)}) + subprocess.check_call(args, env=env) + + def create_yaml(self, yamlfile='', srcdb=False): + dbname = self.srcdb.name if srcdb else self.db.name + args = [self.dbtoyaml] + args.extend(self._db_params()) + if yamlfile: + args.extend(['-o', yamlfile, dbname]) + else: + args.extend(['-r', TEST_DIR, '-m', dbname]) + self.invoke(args) + + def migrate_target(self, yamlfile, outfile): + args = [self.yamltodb] + args.extend(self._db_params()) + if yamlfile: + args.extend(['-u', '-o', outfile, self.db.name, yamlfile]) + else: + args.extend(['-u', '-o', outfile, '-r', TEST_DIR, '-m', + self.db.name]) + self.invoke(args) + + +class AugmentToMapTestCase(PyrseasTestCase): + + def to_map(self, stmts, augmap): + """Apply an augment map and return a map of the updated database. + + :param stmts: list of SQL statements to execute + :param augmap: dictionary describing the augmentations + :return: dictionary of the updated database + """ + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + self.config_options(schemas=[], tables=[], no_owner=True, + no_privs=True, multiple_files=False) + db = AugmentDatabase(self.cfg) + return db.apply(augmap) diff --git a/pyrseas/yamltodb.py b/pyrseas/yamltodb.py new file mode 100644 index 0000000..d0d793c --- /dev/null +++ b/pyrseas/yamltodb.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""yamltodb - generate SQL statements to update a PostgreSQL database +to match the schema specified in a YAML file""" + +from __future__ import print_function +import sys +from argparse import FileType + +import yaml + +from pyrseas import __version__ +from pyrseas.database import Database +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(): + """Convert YAML specifications to database DDL.""" + parser = cmd_parser("Generate SQL statements to update a PostgreSQL " + "database to match the schema specified in a " + "YAML-formatted file(s)", __version__) + parser.add_argument('-m', '--multiple-files', action='store_true', + help='input from multiple files (metadata directory)') + parser.add_argument('spec', nargs='?', type=FileType('r'), + default=sys.stdin, help='YAML specification') + parser.add_argument('-1', '--single-transaction', action='store_true', + dest='onetrans', help="wrap commands in BEGIN/COMMIT") + parser.add_argument('-u', '--update', action='store_true', + help="apply changes to database (implies -1)") + parser.add_argument('--revert', action='store_true', + help="generate SQL to revert changes (experimental)") + parser.add_argument('-n', '--schema', metavar='SCHEMA', dest='schemas', + action='append', default=[], + help="process only named schema(s) (default all)") + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + db = Database(cfg) + if options.multiple_files: + inmap = db.map_from_dir() + else: + try: + inmap = yaml.safe_load(options.spec) + except Exception as exc: + print("Unable to process the input YAML file") + print("Error is '%s'" % exc) + return 1 + + stmts = db.diff_map(inmap) + if stmts: + fd = output or sys.stdout + if options.onetrans or options.update: + print("BEGIN;", file=fd) + for stmt in stmts: + if isinstance(stmt, tuple): + outstmt = "".join(stmt) + '\n' + else: + outstmt = "%s;\n" % stmt + print(outstmt, file=fd) + if options.onetrans or options.update: + print("COMMIT;", file=fd) + if options.update: + try: + for stmt in stmts: + if isinstance(stmt, tuple): + # expected format: (\copy, table, from, path, csv) + db.dbconn.copy_from(stmt[3], stmt[1]) + else: + db.dbconn.execute(stmt) + except: + db.dbconn.rollback() + raise + else: + db.dbconn.commit() + print("Changes applied", file=sys.stderr) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/yamlutil.py b/pyrseas/yamlutil.py new file mode 100644 index 0000000..febed8c --- /dev/null +++ b/pyrseas/yamlutil.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Pyrseas YAML utilities""" + +from yaml import add_representer, dump + +class MultiLineStr(str): + """ Marker for multiline strings""" + + +def MultiLineStr_presenter(dumper, data): + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') +add_representer(MultiLineStr, MultiLineStr_presenter) + + +def yamldump(objmap): + """Dump an object map using yaml.dump with certain defaults + + :param objmap: dictionary + :return: dumped object map + """ + return dump(objmap, default_flow_style=False, allow_unicode=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2361c86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +PyYAML>=5.3 +psycopg>=3.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d54b922 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Pyrseas - Utilities to assist with database schema versioning. +""" +import sys + +from setuptools import setup +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + errno = pytest.main(self.test_args) + sys.exit(errno) + + +setup( + name='Pyrseas', + version='0.10.0', + packages=['pyrseas', 'pyrseas.dbobject', 'pyrseas.lib', 'pyrseas.augment', + ], + package_data={'pyrseas': ['config.yaml']}, + entry_points={ + 'console_scripts': [ + 'dbtoyaml = pyrseas.dbtoyaml:main', + 'yamltodb = pyrseas.yamltodb:main', + 'dbaugment = pyrseas.dbaugment:main']}, + + install_requires=[ + 'psycopg >= 3.1', + 'PyYAML >= 5.3'], + + tests_require=['pytest'], + cmdclass={'test': PyTest}, + + author='Joe Abbate', + author_email='jma@freedomcircle.com', + description='Utilities to assist in database schema versioning', + long_description=open('README.rst').read(), + url='https://perseas.github.io/', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: SQL', + 'Topic :: Database :: Front-Ends', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Version Control'], + platforms='OS-independent', + license='BSD') diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..db22637 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# to avoid setuptools/distutils bug diff --git a/tests/augment/test_audit.py b/tests/augment/test_audit.py new file mode 100644 index 0000000..8107290 --- /dev/null +++ b/tests/augment/test_audit.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- +"""Test audit columns""" + +import pytest + +from pyrseas.testutils import AugmentToMapTestCase + +CREATE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +FUNC_SRC1 = """ +BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP; + RETURN NEW; +END""" + +FUNC_SRC2 = """ +BEGIN + NEW.updated = CURRENT_TIMESTAMP; + RETURN NEW; +END""" + + +class AuditColumnsTestCase(AugmentToMapTestCase): + """Test mapping of audit column augmentations""" + + def test_predef_column(self): + "Add predefined audit column" + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'created_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'created_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_unknown_table(self): + "Error on non-existent table" + augmap = {'schema sd': {'table t2': { + 'audit_columns': 'created_date_only'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_bad_audit_spec(self): + "Error on bad audit column specification" + augmap = {'schema sd': {'table t1': { + 'audit_column': 'created_date_only'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_unknown_audit_spec(self): + "Error on non-existent audit column specification" + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'created_date'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_new_column(self): + "Add new (non-predefined) audit column" + augmap = {'augmenter': {'columns': { + 'modified_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}, + 'audit_columns': {'modified_date_only': { + 'columns': ['modified_date']}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'modified_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_rename_column(self): + "Add predefined audit column but with new name" + augmap = {'augmenter': {'columns': { + 'modified_timestamp': {'name': 'updated'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'modified_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + colmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'updated': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_modified_only': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.audit_modified', 'timing': 'before'}}} + funcmap = {'language': 'plpgsql', 'returns': 'trigger', + 'security_definer': True, 'description': + 'Provides modified_timestamp values for audit columns.', + 'source': FUNC_SRC2} + assert dbmap['schema sd']['table t1'] == colmap + assert dbmap['schema sd']['function audit_modified()'] == funcmap + + def test_change_column_type(self): + "Add predefined audit column but with changed datatype" + augmap = {'augmenter': {'columns': {'created_date': {'type': 'text'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'created_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'created_date': {'type': 'text', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_columns_with_trigger(self): + "Add predefined audit columns with trigger" + augmap = {'schema sd': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_default': { + 'events': ['update'], 'level': 'row', + 'procedure': 'sd.audit_default', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function audit_default()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function audit_default()'][ + 'source'] == FUNC_SRC1 + + def test_nondefault_schema_with_trigger(self): + "Add predefined audit columns with trigger in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text)"] + augmap = {'schema s1': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_default': { + 'events': ['update'], 'level': 'row', + 'procedure': 's1.audit_default', 'timing': 'before'}}} + assert expmap == dbmap['schema s1']['table t1'] + assert dbmap['schema s1']['function audit_default()']['returns'] == \ + 'trigger' + assert dbmap['schema s1']['function audit_default()'][ + 'source'] == FUNC_SRC1 + + def test_skip_existing_columns(self): + "Do not add already existing audit columns" + stmts = [CREATE_STMT, + "ALTER TABLE t1 ADD modified_by_user varchar(63) NOT NULL", + "ALTER TABLE t1 ADD modified_timestamp " + "timestamp with time zone NOT NULL"] + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}] + assert expmap == dbmap['schema sd']['table t1']['columns'] + + def test_change_existing_columns(self): + "Change already existing audit columns" + stmts = [CREATE_STMT, "ALTER TABLE t1 ADD modified_by_user text ", + "ALTER TABLE t1 ADD modified_timestamp " + "timestamp with time zone NOT NULL"] + augmap = {'schema sd': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}] + assert expmap == dbmap['schema sd']['table t1']['columns'] + + def test_custom_function_template(self): + "Add new (non-predefined) audit trigger using a function template" + template = """ + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + source = """ + BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + augmap = { + 'augmenter': { + 'audit_columns': {'custom': { + 'columns': ['modified_by_user', 'modified_timestamp'], + 'triggers': ['custom_audit']}}, + 'function_templates': {'custom_template': template}, + 'functions': {'custom_audit()': { + 'description': 'Maintain custom audit columns', + 'language': 'plpgsql', + 'returns': 'trigger', + 'security_definer': True, + 'source': '{{custom_template}}'}}, + 'triggers': {'custom_audit': { + 'events': ['insert', 'update'], + 'level': 'row', + 'name': '{{table_name}}_20_custom_audit', + 'procedure': 'custom_audit', + 'timing': 'before'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'custom'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_custom_audit': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.custom_audit', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function custom_audit()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function custom_audit()'][ + 'source'] == source + + def test_custom_function_inline_with_column_substitution(self): + "Add new (non-predefined) audit trigger using an inline definition" + template = """ + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + source = """ + BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + augmap = { + 'augmenter': { + 'audit_columns': {'custom': { + 'columns': ['modified_by_user', 'modified_timestamp'], + 'triggers': ['custom_audit']}}, + 'functions': {'custom_audit()': { + 'description': 'Maintain custom audit columns', + 'language': 'plpgsql', + 'returns': 'trigger', + 'security_definer': True, + 'source': template}}, + 'triggers': {'custom_audit': { + 'events': ['insert', 'update'], + 'level': 'row', + 'name': '{{table_name}}_20_custom_audit', + 'procedure': 'custom_audit', + 'timing': 'before'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'custom'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_custom_audit': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.custom_audit', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function custom_audit()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function custom_audit()'][ + 'source'] == source diff --git a/tests/dbobject/__init__.py b/tests/dbobject/__init__.py new file mode 100644 index 0000000..b0bfa64 --- /dev/null +++ b/tests/dbobject/__init__.py @@ -0,0 +1 @@ +"""Pyrseas dbobject unit tests""" diff --git a/tests/dbobject/test_cast.py b/tests/dbobject/test_cast.py new file mode 100644 index 0000000..1b6af85 --- /dev/null +++ b/tests/dbobject/test_cast.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +"""Test casts""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +SOURCE = "SELECT CAST($1::int AS boolean)" +CREATE_FUNC = "CREATE FUNCTION int2_bool(smallint) RETURNS boolean " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE +CREATE_DOMAIN = "CREATE DOMAIN d1 AS integer" +CREATE_STMT1 = "CREATE CAST (smallint AS boolean) WITH FUNCTION " \ + "sd.int2_bool(smallint)" +CREATE_STMT3 = "CREATE CAST (d1 AS integer) WITH INOUT AS IMPLICIT" +DROP_STMT = "DROP CAST IF EXISTS (smallint AS boolean)" +COMMENT_STMT = "COMMENT ON CAST (smallint AS boolean) IS 'Test cast 1'" + + +class CastToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing casts""" + + def test_map_cast_function(self): + "Map a cast with a function" + dbmap = self.to_map([CREATE_FUNC, CREATE_STMT1], superuser=True) + expmap = {'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'} + assert dbmap['cast (smallint as boolean)'] == expmap + + def test_map_cast_inout(self): + "Map a cast with INOUT" + dbmap = self.to_map([CREATE_DOMAIN, CREATE_STMT3]) + expmap = {'context': 'implicit', 'method': 'inout', + 'depends_on': ['domain d1']} + assert dbmap['cast (sd.d1 as integer)'] == expmap + + def test_map_cast_comment(self): + "Map a cast comment" + dbmap = self.to_map([CREATE_FUNC, CREATE_STMT1, COMMENT_STMT], + superuser=True) + assert dbmap['cast (smallint as boolean)']['description'] == \ + 'Test cast 1' + + +class CastToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input casts""" + + def test_create_cast_function(self): + "Create a cast with a function" + stmts = [DROP_STMT, CREATE_FUNC] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == CREATE_STMT1 + + def test_create_cast_inout(self): + "Create a cast with INOUT" + stmts = [CREATE_DOMAIN, "DROP CAST IF EXISTS (d1 AS integer)"] + inmap = self.std_map() + inmap.update({'cast (d1 as integer)': { + 'context': 'implicit', 'method': 'inout'}}) + inmap['schema sd'].update({'domain d1': {'type': 'integer'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == CREATE_STMT3 + + def test_create_cast_schema(self): + "Create a cast using a type/domain in a non-default schema" + stmts = ["CREATE SCHEMA s1", "CREATE DOMAIN s1.d1 AS integer", + "DROP CAST IF EXISTS (integer AS s1.d1)"] + inmap = self.std_map() + inmap.update({'cast (integer as s1.d1)': { + 'context': 'assignment', 'method': 'binary coercible'}}) + inmap.update({'schema s1': {'domain d1': {'type': 'integer'}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "CREATE CAST (integer AS s1.d1) " \ + "WITHOUT FUNCTION AS ASSIGNMENT" + + def test_bad_cast_map(self): + "Error creating a cast with a bad map" + inmap = self.std_map() + inmap.update({'(smallint as boolean)': { + 'function': 'int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_cast(self): + "Drop an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP CAST (smallint AS boolean)" + + def test_cast_with_comment(self): + "Create a cast with a comment" + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Test cast 1', 'function': 'sd.int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + sql = self.to_sql(inmap, [DROP_STMT]) + # sql[0:1] -> SET, CREATE FUNCTION + assert fix_indent(sql[2]) == CREATE_STMT1 + assert sql[3] == COMMENT_STMT + + def test_comment_on_cast(self): + "Create a comment for an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Test cast 1', 'function': 'int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_cast_comment(self): + "Drop a comment on an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1, COMMENT_STMT] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'function': 'int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + assert self.to_sql(inmap, stmts, superuser=True) == \ + ["COMMENT ON CAST (smallint AS boolean) IS NULL"] + + def test_change_cast_comment(self): + "Change existing comment on a cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1, COMMENT_STMT] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Changed cast 1', 'function': 'int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + assert self.to_sql(inmap, stmts, superuser=True) == \ + ["COMMENT ON CAST (smallint AS boolean) IS 'Changed cast 1'"] + + def test_cast_function_view_depends(self): + "Cast that depends on a function that depends on a view. See #86" + stmts = ["CREATE TABLE t1 (id integer)"] + inmap = self.std_map() + inmap.update({'cast (sd.v1 as sd.t1)': { + 'context': 'explicit', 'function': 'sd.v1_to_t1(sd.v1)', + 'method': 'function'}}) + inmap['schema sd'].update({ + 'function v1_to_t1(sd.v1)': { + 'returns': 'sd.t1', 'language': 'plpgsql', + 'source': "\nDECLARE o sd.t1;\nBEGIN o:= ROW($1.id)::t1;\n" + "RETURN o;\nEND"}, + 'table t1': {'columns': [{'id': {'type': 'integer'}}]}, + 'view v1': {'definition': " SELECT t1.id\n FROM sd.t1;", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 3 + assert fix_indent(sql[0]) == \ + "CREATE VIEW sd.v1 AS SELECT t1.id FROM sd.t1" + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.v1_to_t1(sd.v1) " \ + "RETURNS sd.t1 LANGUAGE plpgsql AS $_$\nDECLARE o sd.t1;\n" \ + "BEGIN o:= ROW($1.id)::t1;\nRETURN o;\nEND$_$" + assert fix_indent(sql[2]) == "CREATE CAST (sd.v1 AS sd.t1) WITH " \ + "FUNCTION sd.v1_to_t1(sd.v1)" diff --git a/tests/dbobject/test_collation.py b/tests/dbobject/test_collation.py new file mode 100644 index 0000000..bcfa1ad --- /dev/null +++ b/tests/dbobject/test_collation.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +"""Test collations + +These tests require that the locale fr_FR.utf8 (or equivalent) be installed. +""" +import sys + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +if sys.platform == 'win32': + COLL = 'French_France.1252' +else: + COLL = 'fr_FR.UTF-8' + +CREATE_STMT = "CREATE COLLATION sd.c1 (LC_COLLATE = '%s', LC_CTYPE = '%s')" % ( + COLL, COLL) +COMMENT_STMT = "COMMENT ON COLLATION sd.c1 IS 'Test collation c1'" + + +class CollationToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing collations""" + + def test_map_collation1(self): + "Map a collation" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'lc_collate': COLL, 'lc_ctype': COLL} + assert dbmap['schema sd']['collation c1'] == expmap + + def test_map_collation_comment(self): + "Map a collation comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['collation c1']['description'] == \ + 'Test collation c1' + + def test_map_column_collation(self): + "Map a table with a column collation" + dbmap = self.to_map( + [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text COLLATE c1)"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text', 'collation': 'c1'}}], + 'depends_on': ['collation c1']} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_collation(self): + "Map an index with column collation" + stmts = [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE INDEX t1_idx ON t1 (c2 COLLATE c1)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': { + 'keys': [{'c2': {'collation': 'sd.c1'}}], + 'depends_on': ['collation c1']}}} + assert dbmap['schema sd']['table t1'] == expmap + + +class CollationToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input collations""" + + def test_create_collation1(self): + "Create a collation" + inmap = self.std_map() + inmap['schema sd'].update({'collation c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_collation_schema(self): + "Create a collation in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'collation c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE COLLATION s1.c1 (" \ + "LC_COLLATE = '%s', LC_CTYPE = '%s')" % (COLL, COLL) + + def test_bad_collation_map(self): + "Error creating a collation with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_collation(self): + "Drop an existing collation" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql[0] == "DROP COLLATION sd.c1" + + def test_collation_with_comment(self): + "Create a collation with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'collation c1': { + 'description': 'Test collation c1', + 'lc_collate': COLL, 'lc_ctype': COLL}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_create_table_column_collation(self): + "Create a table with a column with non-default collation" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text', 'collation': 'c1'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (" \ + 'c1 integer NOT NULL, c2 text COLLATE "c1")' + + def test_create_index_collation(self): + "Create an index with column collation" + stmts = [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{'c2': {'collation': 'c1'}}]}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "CREATE INDEX t1_idx ON sd.t1 (c2 COLLATE c1)" + + def test_create_type_attribute_collation(self): + "Create a composite type with an attribute with non-default collation" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'text', 'collation': 'c1'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TYPE sd.t1 AS (x integer, " \ + 'y text COLLATE "c1")' diff --git a/tests/dbobject/test_column.py b/tests/dbobject/test_column.py new file mode 100644 index 0000000..d7b01d9 --- /dev/null +++ b/tests/dbobject/test_column.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +"""Test columns""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +TYPELIST = [ + ('SMALLINT', 'smallint'), + ('INTEGER', 'integer'), + ('BIGINT', 'bigint'), + ('int', 'integer'), + ('int2', 'smallint'), + ('int4', 'integer'), + ('int8', 'bigint'), + ('NUMERIC', 'numeric'), + ('NUMERIC(1)', 'numeric(1,0)'), + ('NUMERIC(12)', 'numeric(12,0)'), + ('NUMERIC(1000)', 'numeric(1000,0)'), + ('NUMERIC(12,2)', 'numeric(12,2)'), + ('NUMERIC(1000,500)', 'numeric(1000,500)'), + ('DECIMAL', 'numeric'), + ('dec(9,5)', 'numeric(9,5)'), + ('REAL', 'real'), + ('DOUBLE PRECISION', 'double precision'), + ('FLOAT', 'double precision'), + ('FLOAT(1)', 'real'), + ('FLOAT(24)', 'real'), + ('FLOAT(25)', 'double precision'), + ('FLOAT(53)', 'double precision'), + # SERIAL and BIGSERIAL have side effects + ('MONEY', 'money'), + ('CHARACTER(1)', 'character(1)'), + ('CHARACTER VARYING(200000)', 'character varying(200000)'), + ('CHAR(16)', 'character(16)'), + ('VARCHAR(256)', 'character varying(256)'), + ('TEXT', 'text'), + ('CHAR', 'character(1)'), + ('CHARACTER VARYING', 'character varying'), + ('"char"', '"char"'), + ('name', 'name'), + ('bytea', 'bytea'), + ('DATE', 'date'), + ('TIME', 'time without time zone'), + ('TIME WITHOUT TIME ZONE', 'time without time zone'), + ('TIME WITH TIME ZONE', 'time with time zone'), + ('TIMESTAMP', 'timestamp without time zone'), + ('TIMESTAMP WITHOUT TIME ZONE', 'timestamp without time zone'), + ('TIMESTAMP WITH TIME ZONE', 'timestamp with time zone'), + ('TIME(0)', 'time(0) without time zone'), + ('TIME(1) WITHOUT TIME ZONE', 'time(1) without time zone'), + ('TIME(2) WITH TIME ZONE', 'time(2) with time zone'), + ('TIMESTAMP(3)', 'timestamp(3) without time zone'), + ('TIMESTAMP(4) WITHOUT TIME ZONE', 'timestamp(4) without time zone'), + ('TIMESTAMP(5) WITH TIME ZONE', 'timestamp(5) with time zone'), + ('INTERVAL', 'interval'), + ('INTERVAL(6)', 'interval(6)'), + ('INTERVAL YEAR', 'interval year'), + ('INTERVAL MONTH', 'interval month'), + ('INTERVAL DAY', 'interval day'), + ('INTERVAL HOUR', 'interval hour'), + ('INTERVAL MINUTE', 'interval minute'), + ('INTERVAL SECOND', 'interval second'), + ('INTERVAL YEAR TO MONTH', 'interval year to month'), + ('INTERVAL DAY TO HOUR', 'interval day to hour'), + ('INTERVAL DAY TO MINUTE', 'interval day to minute'), + ('INTERVAL DAY TO SECOND', 'interval day to second'), + ('INTERVAL HOUR TO MINUTE', 'interval hour to minute'), + ('INTERVAL HOUR TO SECOND', 'interval hour to second'), + ('INTERVAL MINUTE TO SECOND', 'interval minute to second'), + ('INTERVAL SECOND(3)', 'interval second(3)'), + ('INTERVAL HOUR TO SECOND(5)', 'interval hour to second(5)'), + ('BOOLEAN', 'boolean'), + ('POINT', 'point'), + ('LINE', 'line'), + ('LSEG', 'lseg'), + ('BOX', 'box'), + ('PATH', 'path'), + ('POLYGON', 'polygon'), + ('CIRCLE', 'circle'), + ('cidr', 'cidr'), + ('inet', 'inet'), + ('macaddr', 'macaddr'), + ('BIT(2)', 'bit(2)'), + ('BIT VARYING(100)', 'bit varying(100)'), + ('BIT', 'bit(1)'), + ('BIT VARYING', 'bit varying'), + ('tsvector', 'tsvector'), + ('tsquery', 'tsquery'), + ('UUID', 'uuid'), + ('XML', 'xml'), + ('JSON', 'json'), + ('JSONB', 'jsonb')] + +CREATE_STMT1 = "CREATE TABLE t1 (c1 integer, c2 text)" +CREATE_STMT2 = "CREATE TABLE t1 (c1 integer, c2 text, c3 date)" +CREATE_STMT3 = "CREATE TABLE t1 (c1 integer, c2 text, c3 date, c4 text)" +DROP_COL_STMT = "ALTER TABLE t1 DROP COLUMN c3" + + +class ColumnToMapTestCase(DatabaseToMapTestCase): + """Test mapping of column-related elements in created tables""" + + def test_data_types(self): + "Map a table with columns for (almost) each native PostgreSQL type" + colstab = [] + colsmap = [] + for colnum, (coltype, maptype) in enumerate(TYPELIST): + col = "c%d" % (colnum + 1) + colstab.append("%s %s" % (col, coltype)) + if coltype == 'name' and self.db.version >= 120000: + colsmap.append({col: {'type': maptype, 'collation': 'C'}}) + else: + colsmap.append({col: {'type': maptype}}) + dbmap = self.to_map(["CREATE TABLE t1 (%s)" % ", ".join(colstab)]) + expmap = {'columns': colsmap} + assert dbmap['schema sd']['table t1'] == expmap + + def test_not_null(self): + "Map a table with a NOT NULL column" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 INTEGER NULL, " + "c3 INTEGER NOT NULL)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'integer', 'not_null': True}}]} + + assert dbmap['schema sd']['table t1'] == expmap + + def test_column_defaults(self): + "Map a table with various types and each with a DEFAULT clause" + stmts = ["CREATE TABLE t1 (c1 INTEGER DEFAULT 12345, " + "c2 NUMERIC DEFAULT 98.76, c3 REAL DEFAULT 15e-2, " + "c4 TEXT DEFAULT 'Abc def', c5 DATE DEFAULT CURRENT_DATE, " + "c6 TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, " + "c7 BOOLEAN DEFAULT FALSE)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [ + {'c1': {'type': 'integer', 'default': '12345'}}, + {'c2': {'type': 'numeric', 'default': '98.76'}}, + {'c3': {'type': 'real', 'default': '0.15'}}, + {'c4': {'type': 'text', 'default': "'Abc def'::text"}}, + {'c5': {'type': 'date', 'default': "CURRENT_DATE"}}, + {'c6': {'type': 'timestamp with time zone', 'default': + "CURRENT_TIMESTAMP"}}, + {'c7': {'type': 'boolean', 'default': 'false'}}]} + assert dbmap['schema sd']['table t1'] == expmap + + def test_statistics(self): + "Map a table with column statistics" + stmts = [CREATE_STMT1, "ALTER TABLE t1 ALTER c1 SET STATISTICS 100"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'statistics': 100}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_identity(self): + "Map a table with an IDENTITY column" + stmts = ["CREATE TABLE t1 (c1 integer GENERATED ALWAYS AS IDENTITY, " + "c2 text)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'identity': 'always'}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + expmap = {'data_type': 'integer', 'start_value': 1, 'increment_by': 1, + 'max_value': 2147483647, 'min_value': None, 'cache_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'} + assert dbmap['schema sd']['sequence t1_c1_seq'] == expmap + + +class ColumnToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of column-related statements from input schemas""" + + def test_create_table_with_defaults(self): + "Create a table with two column DEFAULTs, one referring to a SEQUENCE" + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'default': + "nextval('s1.t1_c1_seq'::regclass)"}}, + {'c2': {'type': 'text', 'not_null': True, + 'collation': 'en_US.utf8'}}, + {'c3': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]}, + 'sequence t1_c1_seq': { + 'cache_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE SEQUENCE s1.t1_c1_seq " \ + "START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" + assert fix_indent(sql[1]) == "CREATE TABLE s1.t1 (c1 integer " \ + "NOT NULL DEFAULT nextval('s1.t1_c1_seq'::regclass), c2 text " \ + 'NOT NULL COLLATE "en_US.utf8", c3 date NOT NULL ' \ + "DEFAULT ('now'::text)::date)" + assert sql[2] == "ALTER SEQUENCE s1.t1_c1_seq OWNED BY s1.t1.c1" + + def test_create_column_identity(self): + "Create a table with a column GENERATED AS IDENTITY" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'identity': 'by default'}}, + {'c2': {'type': 'text'}}]}, + 'sequence t1_c1_seq': { + 'cache_value': 1, 'data_type': 'integer', 'increment_by': 1, + 'max_value': None, 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}) + sql = self.to_sql(inmap) + assert len(sql) == 1 + assert fix_indent(sql[0]) == ( + "CREATE TABLE sd.t1 (c1 integer NOT NULL GENERATED BY DEFAULT AS " + "IDENTITY (SEQUENCE NAME sd.t1_c1_seq START WITH 1 INCREMENT BY 1 " + "NO MINVALUE NO MAXVALUE CACHE 1), c2 text)") + + def test_change_column_default(self): + "Change the default value for an existing column" + stmt = "CREATE TABLE t1 (c1 integer, c2 boolean default true)" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'boolean', 'default': 'false'}}]}}) + sql = self.to_sql(inmap, [stmt]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 SET DEFAULT false" + + def test_set_column_not_null(self): + "Change a nullable column to NOT NULL" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 SET NOT NULL" + + def test_change_column_types(self): + "Change the datatypes of two columns" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'bigint'}}, + {'c2': {'type': 'varchar(25)'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 TYPE bigint" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 TYPE varchar(25)" + + def test_add_column1(self): + "Add new column to a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}, {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_add_column2(self): + "Add column to a table that has a dropped column" + stmts = [CREATE_STMT2, "ALTER TABLE t1 DROP COLUMN c2"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c2 text" + + def test_add_column3(self): + "No change on a table that has a dropped column" + stmts = [CREATE_STMT3, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 0 + + def test_add_column4(self): + "Add two columns to a table that has a dropped column" + stmts = [CREATE_STMT2, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}, {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 date" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_drop_column1(self): + "Drop a column from the end of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c4" + + def test_drop_column2(self): + "Drop a column from the middle of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c3" + + def test_drop_column3(self): + "Drop a column from the beginning of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_rename_column(self): + "Rename a table column" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c3': {'type': 'text', 'oldname': 'c2'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert sql[0] == "ALTER TABLE sd.t1 RENAME COLUMN c2 TO c3" + + def test_drop_add_column1(self): + "Drop and re-add table column from the end, almost like a RENAME" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 date" + assert sql[1] == "ALTER TABLE sd.t1 DROP COLUMN c3" + + def test_drop_add_column2(self): + "Drop and re-add table column from the beginning" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + assert sql[1] == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_drop_add_column3(self): + "Drop and re-add table columns from table with dropped column" + stmts = [CREATE_STMT2, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 date" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + assert sql[2] == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_drop_column_in_schema(self): + "Drop a column from a table in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text, c3 date)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE s1.t1 DROP COLUMN c3" + + def test_inherit_add_parent_column(self): + "Add a column to parent table, child should not add as well" + stmts = [CREATE_STMT1, "CREATE TABLE t2 (c3 date) INHERITS (t1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'date'}}, + {'c4': {'type': 'text', 'inherited': True}}], + 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_inherit_drop_parent_column(self): + "Drop a column from a parent table, child should not drop as well" + stmts = [CREATE_STMT1, "CREATE TABLE t2 (c3 date) INHERITS (t1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c3': {'type': 'date'}}], 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c2" + + def test_alter_statistics(self): + "Alter a table to add column statistics" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'statistics': 100}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1, "ALTER TABLE t1 ALTER c2 " + "SET STATISTICS 1000"]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 SET STATISTICS 100" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 SET STATISTICS -1" diff --git a/tests/dbobject/test_constraint.py b/tests/dbobject/test_constraint.py new file mode 100644 index 0000000..16e25a1 --- /dev/null +++ b/tests/dbobject/test_constraint.py @@ -0,0 +1,1275 @@ +# -*- coding: utf-8 -*- +"""Test constraints""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +COMMENT_STMT = "COMMENT ON CONSTRAINT cns1 ON sd.t1 IS 'Test constraint cns1'" + + +class CheckConstraintToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created CHECK constraints""" + + def test_check_constraint_1(self): + "Map a table with a CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 SMALLINT CHECK (c2 < 1000))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'smallint'}}], + 'check_constraints': {'t1_c2_check': { + 'columns': ['c2'], 'expression': '(c2 < 1000)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_2(self): + "Map a table with a two-column, named CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, " + "CONSTRAINT t1_check_ratio CHECK (c2 * 100 / c1 <= 50))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}], + 'check_constraints': {'t1_check_ratio': { + 'columns': ['c2', 'c1'], + 'expression': '(((c2 * 100) / c1) <= 50)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_no_columns(self): + "Map a table with a check constraint without any columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER, " + "CONSTRAINT t1_empty CHECK (false))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}], + 'check_constraints': {'t1_empty': { + 'expression': 'false'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_inherited(self): + "Map a table with an inherited CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_c1_check " + "CHECK (c1 > 0))", "CREATE TABLE t2 (c2 text) INHERITS (t1)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text'}}], + 'inherits': ['t1'], + 'check_constraints': {'t1_c1_check': { + 'columns': ['c1'], 'inherited': True, + 'expression': '(c1 > 0)'}}} + assert dbmap['schema sd']['table t2'] == expmap + + +class CheckConstraintToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input CHECK constraints""" + + def test_create_w_check_constraint(self): + "Create new table with a single column CHECK constraint" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'check_constraints': { + 't1_c1_check': {'columns': ['c1'], + 'expression': '(c1 > 0 and c1 < 1000000)'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c1_check CHECK (c1 > 0 and c1 < 1000000)" + + def test_add_check_constraint(self): + "Add a two-column CHECK constraint to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'check_constraints': { + 't1_check_2_1': {'columns': ['c2', 'c1'], + 'expression': '(c2 != c1)'}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_check_2_1 CHECK (c2 != c1)" + + def test_add_check_constraint_no_columns(self): + "Add a CHECK constraint with no column" + stmts = ["CREATE TABLE t1 (c1 INTEGER)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], + 'check_constraints': { + 't1_empty': {'expression': 'false'}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_empty CHECK (false)" + + def test_add_check_inherited(self): + "Add a table with an inherited CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_c1_check " + "CHECK (c1 > 0))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], 'check_constraints': { + 't1_c1_check': {'columns': ['c1'], 'expression': '(c1 > 0)'}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text'}}], 'check_constraints': { + 't1_c1_check': + {'columns': ['c1'], 'expression': '(c1 > 0)', + 'inherited': True}}, 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t2 (c2 text) " \ + "INHERITS (sd.t1)" + assert len(sql) == 1 + + def test_change_check_constraint(self): + "Change expression of a check constraint in an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_check1 " + "CHECK (c1 > 0), CONSTRAINT t1_check2 CHECK (c1 < 100))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], 'check_constraints': { + 't1_check1': {'columns': ['c1'], 'expression': '(c1 > 10)'}, + 't1_check2': {'columns': ['c1'], 'expression': '(c1 < 100)'} + }} + }) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_check1" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_check1 CHECK (c1 > 10)" + assert len(sql) == 2 + + +class PrimaryKeyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created PRIMARY KEYs""" + + map_pkey1 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}} + map_pkey2 = {'columns': [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(5)', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}} + + map_pkey3 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_prim_key': {'columns': ['c1']}}} + + def test_primary_key_1(self): + "Map a table with a single-column primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey1 + + def test_primary_key_2(self): + "Map a table with a single-column primary key, table-level constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, PRIMARY KEY (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey1 + + def test_primary_key_3(self): + "Map a table with two-column primary key, atypical order" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT, " + "PRIMARY KEY (c2, c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey2 + + def test_primary_key_4(self): + "Map a table with a named primary key constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, " + "CONSTRAINT t1_prim_key PRIMARY KEY (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey3 + + def test_primary_key_5(self): + "Map a table with a named primary key, column level constraint" + stmts = ["CREATE TABLE t1 (" + "c1 INTEGER CONSTRAINT t1_prim_key PRIMARY KEY, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey3 + + def test_primary_key_cluster(self): + "Map a table with a primary key and CLUSTER on it" + stmts = ["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CLUSTER t1 USING t1_pkey"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], 'cluster': True}}} + + def test_map_pk_comment(self): + "Map a primary key with a comment" + stmts = ["CREATE TABLE t1 (c1 integer CONSTRAINT cns1 PRIMARY KEY, " + "c2 text)", COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['primary_key']['cns1'][ + 'description'] == 'Test constraint cns1' + + +class PrimaryKeyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input PRIMARY KEYs""" + + def test_create_with_primary_key(self): + "Create new table with single column primary key" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'text'}}, {'c2': {'type': 'integer'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 text, c2 integer)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_add_primary_key(self): + "Add a two-column primary key to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1, c2)" + + def test_alter_primary_key_add(self): + "Add a two-column primary key to an existing table with primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 INTEGER, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1, c2)" + + def test_alter_primary_key_change(self): + "Change primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 INTEGER, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_alter_primary_key_change_order(self): + "Change primary key" + stmts = ["CREATE TABLE t1 (c2 INTEGER, c1 INTEGER PRIMARY KEY)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_drop_primary_key(self): + "Drop a primary key on an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL PRIMARY KEY, c2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}, + 'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey"] + + def test_primary_key_clustered(self): + "Create new table clustered on the primary key" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_pkey" + + def test_primary_key_uncluster(self): + "Remove cluster from table clustered on the primary key" + stmts = ["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CLUSTER t1 USING t1_pkey"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET WITHOUT CLUSTER" + + def test_change_primary_key(self): + "Changing primary key columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL," + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1)" + assert len(sql) == 2 + + def test_change_order_primary_key(self): + "Changing primary key columns order" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL," + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2, c1)" + assert len(sql) == 2 + + +class ForeignKeyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created FOREIGN KEYs""" + + map_fkey1 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], + 'on_update': 'cascade', + 'on_delete': 'restrict', + 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc1'] + } + }}} + + map_fkey2 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + # PG 12 uses all column names to generate the constraint's name + map_fkey2_12 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_c3_c4_fkey': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + map_fkey3 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_fgn_key': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + map_fkey4 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(5)', + 'not_null': True}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'primary_key': {'t1_prim_key': {'columns': ['c1', 'c2']}}, + 'foreign_keys': { + 't1_fgn_key1': { + 'columns': ['c2', 'c3', 'c4'], + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}, + 't1_fgn_key2': {'columns': ['c2'], + 'references': {'schema': 'sd', + 'table': 't3', + 'columns': ['qc1']}}}} + + def test_foreign_key_1(self): + "Map a table with a single-column foreign key on another table" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER REFERENCES t2 (pc1) " + "ON UPDATE CASCADE ON DELETE RESTRICT" + ", c3 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey1 + + def test_foreign_key_2(self): + "Map a table with a single-column foreign key, table level constraint" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, c3 TEXT, " + "FOREIGN KEY (c2) REFERENCES t2 (pc1)" + "ON UPDATE CASCADE ON DELETE RESTRICT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey1 + + def test_foreign_key_3(self): + "Map a table with a three-column foreign key" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "FOREIGN KEY (c2, c3, c4) REFERENCES t2 (pc2, pc1, pc3))"] + dbmap = self.to_map(stmts) + if self.db.version < 120000: + assert dbmap['schema sd']['table t1'] == self.map_fkey2 + else: + assert dbmap['schema sd']['table t1'] == self.map_fkey2_12 + + def test_foreign_key_4(self): + "Map a table with a named, three-column foreign key" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "CONSTRAINT t1_fgn_key FOREIGN KEY (c2, c3, c4) " + "REFERENCES t2 (pc2, pc1, pc3))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey3 + + def test_foreign_key_5(self): + "Map a table with a primary key and two foreign keys" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t3 (qc1 CHAR(5) PRIMARY KEY, qc2 text)", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "CONSTRAINT t1_prim_key PRIMARY KEY (c1, c2), " + "CONSTRAINT t1_fgn_key1 FOREIGN KEY (c2, c3, c4) " + "REFERENCES t2 (pc2, pc1, pc3), " + "CONSTRAINT t1_fgn_key2 FOREIGN KEY (c2) " + "REFERENCES t3 (qc1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey4 + + def test_foreign_key_actions(self): + "Map a table with foreign key ON UPDATE/ON DELETE actions" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, c3 TEXT, " + "FOREIGN KEY (c2) REFERENCES t2 (pc1) " + "ON UPDATE RESTRICT ON DELETE SET NULL)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'on_update': 'restrict', + 'on_delete': 'set null', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_cross_schema_foreign_key(self): + "Map a table with a foreign key on a table in another schema" + stmts = ["CREATE SCHEMA s1", "CREATE SCHEMA s2", + "CREATE TABLE s2.t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE s1.t1 (c1 INTEGER PRIMARY KEY, " + "c2 INTEGER REFERENCES s2.t2 (pc1), c3 TEXT)"] + dbmap = self.to_map(stmts) + t2map = {'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['pc1']}}} + t1map = {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}, + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 's2', 'table': 't2', + 'columns': ['pc1']}}}}} + assert dbmap['schema s2']['table t2'] == t2map + assert dbmap['schema s1'] == t1map + + def test_multiple_foreign_key(self): + "Map a table with its primary key referenced by two others" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, " + "c2 integer REFERENCES t1 (pc1), c3 text, " + "c4 integer REFERENCES t1 (pc1))"] + dbmap = self.to_map(stmts) + t1map = {'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['pc1']}}} + t2map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}, + {'c4': {'type': 'integer'}}], + 'foreign_keys': { + 't2_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}, + 't2_c4_fkey': { + 'columns': ['c4'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == t1map + assert dbmap['schema sd']['table t2'] == t2map + + def test_foreign_key_dropped_column(self): + "Map a table with a foreign key after a column has been dropped" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, c2 text, c3 smallint, " + "c4 integer REFERENCES t1 (pc1))", + "ALTER TABLE t2 DROP COLUMN c3"] + dbmap = self.to_map(stmts) + t2map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c4': {'type': 'integer'}}], + 'foreign_keys': {'t2_c4_fkey': { + 'columns': ['c4'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t2'] == t2map + + def test_foreign_key_deferred(self): + "Check constraints deferred status" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, " + "c2 INTEGER REFERENCES t2 (pc1), " + "c3 INTEGER REFERENCES t2 (pc1) DEFERRABLE, " + "c4 INTEGER REFERENCES t2 (pc1) DEFERRABLE " + "INITIALLY DEFERRED)"] + dbmap = self.to_map(stmts) + fks = dbmap['schema sd']['table t1']['foreign_keys'] + assert not fks['t1_c2_fkey'].get('deferrable') + assert not fks['t1_c2_fkey'].get('deferred') + assert fks['t1_c3_fkey'].get('deferrable') + assert not fks['t1_c3_fkey'].get('deferred') + assert fks['t1_c4_fkey'].get('deferrable') + assert fks['t1_c4_fkey'].get('deferred') + + def test_map_foreign_key_match(self): + "Map a foreign key constraint with a MATCH specification" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, " + "c2 INTEGER REFERENCES t2 (pc1) MATCH FULL, c3 TEXT)"] + dbmap = self.to_map(stmts) + t1map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == t1map + + def test_map_fk_comment(self): + "Map a foreign key with a comment" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER " + "CONSTRAINT cns1 REFERENCES t2 (pc1), c3 TEXT)", COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['foreign_keys']['cns1'][ + 'description'] == 'Test constraint cns1' + + +class ForeignKeyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input FOREIGN KEYs""" + + def test_create_with_foreign_key(self): + "Create a table with a foreign key constraint" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap) + # can't control which table will be created first + crt1 = 0 + crt2 = 1 + if 't1' in sql[1]: + crt1 = 1 + crt2 = 0 + assert fix_indent(sql[crt1]) == \ + "CREATE TABLE sd.t1 (c11 integer, c12 text)" + assert fix_indent(sql[crt2]) == \ + "CREATE TABLE sd.t2 (c21 integer, c22 text, c23 integer)" + assert fix_indent(sql[2]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c11)" + assert fix_indent(sql[3]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + + def test_create_foreign_key_deferred(self): + "Create a table with various foreign key deferring constraints" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}], + 'unique_constraints': {'t1_c11_key': {'columns': ['c11']}}}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}, + {'c24': {'type': 'integer'}}, + {'c25': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}, + 't2_c24_fkey': { + 'columns': ['c24'], + 'references': {'columns': ['c11'], + 'table': 't1'}, + 'deferrable': True}, + 't2_c25_fkey': { + 'columns': ['c25'], + 'references': {'columns': ['c11'], + 'table': 't1'}, + 'deferrable': True, 'deferred': True}}}}) + sql = self.to_sql(inmap) + assert len(sql) == 6 + # can't control which table/constraint will be created first + sql[0:2] = list(sorted(sql[0:2])) + sql[2:5] = list(sorted(sql[2:5])) + assert fix_indent(sql[0]) == \ + "CREATE TABLE sd.t1 (c11 integer, c12 text)" + assert fix_indent(sql[1]) == "CREATE TABLE sd.t2 (c21 integer, " \ + "c22 text, c23 integer, c24 integer, c25 integer)" + assert fix_indent(sql[2]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c11_key UNIQUE (c11)" + assert fix_indent(sql[3]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + assert fix_indent(sql[4]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c24_fkey FOREIGN KEY (c24) REFERENCES sd.t1 (c11) DEFERRABLE" + assert fix_indent(sql[5]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c25_fkey FOREIGN KEY (c25) REFERENCES sd.t1 (c11) " \ + "DEFERRABLE INITIALLY DEFERRED" + + def test_add_foreign_key(self): + "Add a two-column foreign key to an existing table" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, " + "c12 INTEGER NOT NULL, c13 TEXT, PRIMARY KEY (c11, c12))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL, " + "c22 TEXT, c23 INTEGER, c24 INTEGER, PRIMARY KEY (c21))"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}, + {'c13': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11', 'c12']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}, + {'c24': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23', 'c24'], + 'references': {'columns': ['c11', 'c12'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23, c24) REFERENCES sd.t1 (c11, c12)" + + def test_alter_foreign_key1(self): + "Change foreign key: referencing column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY NOT NULL, " + "c12 INTEGER NOT NULL)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY NOT NULL, " + "c22 INTEGER, c23 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}, + {'c23': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + + def test_alter_foreign_key_columns(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL UNIQUE, " + "c12 INTEGER NOT NULL UNIQUE)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY NOT NULL, " + "c22 INTEGER, c23 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}], + 'unique_constraints': { + 't1_c11_key': {'columns': ['c11']}, + 't1_c12_key': {'columns': ['c12']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}, + {'c23': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'references': {'columns': ['c12'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c12)" + + def test_alter_foreign_key_change_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11) ON UPDATE RESTRICT"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'on_update': 'cascade', + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE" + + def test_alter_foreign_key_add_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'on_update': 'cascade', + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE" + + def test_alter_foreign_key_drop_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11) ON UPDATE RESTRICT"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11)" + + def test_drop_foreign_key(self): + "Drop a foreign key on an existing table" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, c12 TEXT, " + "PRIMARY KEY (c11))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL PRIMARY KEY, " + "c22 INTEGER NOT NULL REFERENCES t1 (c11), c23 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer', 'not_null': True}}, + {'c23': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey"] + + def test_drop_foreign_key_and_column(self): + "Drop a foreign key on an existing table and its referred column" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, c12 TEXT, " + "PRIMARY KEY (c11))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL PRIMARY KEY, " + "c22 INTEGER NOT NULL REFERENCES t1 (c11), c23 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c23': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert sql[1] == "ALTER TABLE sd.t2 DROP COLUMN c22" + + def test_create_foreign_key_actions(self): + "Create a table with foreign key ON UPDATE/ON DELETE actions" + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}]}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], 'on_update': 'cascade', + 'on_delete': 'set default', + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap) + # won't check CREATE TABLEs explicitly here (see first test instead) + assert fix_indent(sql[2]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE ON DELETE SET DEFAULT" + + def test_foreign_key_match(self): + "Create a foreign key constraint with a MATCH specification" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}}, + 'table t2': {'columns': [{'pc1': {'type': 'integer', + 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['pc1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c2_fkey FOREIGN KEY (c2) REFERENCES sd.t2 (pc1) MATCH FULL" + + def test_change_columns_foreign_key(self): + "Changing foreign key columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1) " + "REFERENCES t1 (c1) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c2) REFERENCES sd.t1 (c1) MATCH FULL" + assert len(sql) == 2 + + def test_change_ref_columns_foreign_key(self): + "Changing foreign key reference columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1, c2) " + "REFERENCES t1 (c1, c2) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1', 'c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c2', 'c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1, c2) REFERENCES sd.t1 (c2, c1) MATCH FULL" + + def test_change_match_foreign_key(self): + "Changing foreign key match" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1, c2) " + "REFERENCES t1 (c1, c2) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1', 'c2'], 'match': 'simple', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1', 'c2']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1, c2) REFERENCES sd.t1 (c1, c2)" + assert len(sql) == 2 + + def test_change_actions_foreign_key(self): + "Changing foreign key actions" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1) " + "REFERENCES t1 (c1) ON UPDATE CASCADE)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1'], 'on_update': 'restrict', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1) REFERENCES sd.t1 (c1) ON UPDATE RESTRICT" + assert len(sql) == 2 + + +class UniqueConstraintToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created UNIQUE constraints""" + + map_unique1 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1']}}} + + map_unique2 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'text'}}], + 'unique_constraints': { + 't1_c1_c2_key': {'columns': ['c1', 'c2']}}} + + map_unique3 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unique_constraints': { + 't1_unique_key': {'columns': ['c1']}}} + + def test_unique_1(self): + "Map a table with a single-column unique constraint" + dbmap = self.to_map(["CREATE TABLE t1 (c1 INTEGER UNIQUE, c2 TEXT)"]) + assert dbmap['schema sd']['table t1'] == self.map_unique1 + + def test_unique_2(self): + "Map a table with a single-column unique constraint, table level" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, UNIQUE (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique1 + + def test_unique_3(self): + "Map a table with a two-column unique constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT, " + "UNIQUE (c1, c2))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique2 + + def test_unique_4(self): + "Map a table with a named unique constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, " + "CONSTRAINT t1_unique_key UNIQUE (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique3 + + def test_unique_5(self): + "Map a table with a named unique constraint, column level" + stmts = ["CREATE TABLE t1 ( " + "c1 INTEGER CONSTRAINT t1_unique_key UNIQUE, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique3 + + def test_map_unique_cluster(self): + "Map a table with a unique constraint and CLUSTER on it" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text UNIQUE)", + "CLUSTER t1 USING t1_c2_key"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c2_key': { + 'columns': ['c2'], 'cluster': True}}} + + +class UniqueConstraintToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input UNIQUE constraints""" + + def test_create_w_unique_constraint(self): + "Create new table with a single column unique constraint" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c1_key UNIQUE (c1)" + + def test_add_unique_constraint(self): + "Add a two-column unique constraint to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'unique_constraints': {'t1_c2_key': {'columns': ['c2', 'c1'], + 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c2_key UNIQUE (c2, c1)" + + def test_alter_unique_constraint(self): + "Change unique constraint column" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL)", + "ALTER TABLE t1 ADD CONSTRAINT t1_ukey UNIQUE (c1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'unique_constraints': {'t1_ukey': {'columns': ['c2'], + 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_ukey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_ukey UNIQUE (c2)" + + def test_drop_unique_constraint(self): + "Drop a unique constraint on an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL UNIQUE, c2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}, + 'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t1 DROP CONSTRAINT t1_c1_key"] + + def test_create_unique_clustered(self): + "Create new table clustered on the unique constraint index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1'], + 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_c1_key" + + def test_unique_cluster(self): + "Cluster a table on the unique constraint index" + stmts = ["CREATE TABLE t1 (c1 integer UNIQUE, c2 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1'], + 'cluster': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "CLUSTER sd.t1 USING t1_c1_key" + + def test_change_unique_constraint(self): + "Change columns in unique index" + stmts = ["CREATE TABLE t1 (c1 integer UNIQUE, c2 date)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'date'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t1 DROP CONSTRAINT t1_c1_key" + assert sql[1] == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c1_key UNIQUE (c2)" + assert len(sql) == 2 + + def test_change_order_unique_constraint(self): + "Change columns order in unique index" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, " + "CONSTRAINT t1_unique UNIQUE (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_unique': {'columns': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t1 DROP CONSTRAINT t1_unique" + assert sql[1] == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_unique UNIQUE (c2, c1)" + assert len(sql) == 2 + + +class ConstraintCommentTestCase(InputMapToSqlTestCase): + """Test creation of comments on constraints""" + + def test_check_constraint_with_comment(self): + "Create a CHECK constraint with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'check_constraints': {'cns1': { + 'columns': ['c1'], 'expression': '(c1 > 50)', + 'description': 'Test constraint cns1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT cns1 CHECK (c1 > 50)" + assert sql[2] == COMMENT_STMT + + def test_comment_on_primary_key(self): + "Create a comment for an existing primary key" + stmts = ["CREATE TABLE t1 (c1 text CONSTRAINT cns1 PRIMARY KEY, " + "c2 integer)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'text', 'not_null': True}}, + {'c2': {'type': 'integer'}}], + 'primary_key': {'cns1': {'columns': ['c1'], + 'description': 'Test constraint cns1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_foreign_key_comment(self): + "Drop the comment on an existing foreign key" + stmts = ["CREATE TABLE t2 (c21 integer PRIMARY KEY, c22 text)", + "CREATE TABLE t1 (c11 integer, c12 text, " + "c13 integer CONSTRAINT cns1 REFERENCES t2 (c21))", + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}, + 'table t1': { + 'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}, + {'c13': {'type': 'integer'}}], + 'foreign_keys': {'cns1': { + 'columns': ['c13'], + 'references': {'columns': ['c21'], 'table': 't2'}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONSTRAINT cns1 ON sd.t1 IS NULL"] + + def test_change_unique_constraint_comment(self): + "Change existing comment on a unique constraint" + stmts = ["CREATE TABLE t1 (c1 integer CONSTRAINT cns1 UNIQUE, " + "c2 text)", COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'cns1': { + 'columns': ['c1'], + 'description': "Changed constraint cns1"}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONSTRAINT cns1 ON sd.t1 IS " + "'Changed constraint cns1'"] + + def test_constraint_comment_schema(self): + "Add comment on a constraint for a table in another schema" + stmts = ["CREATE SCHEMA s1", "CREATE TABLE s1.t1 (c1 integer " + "CONSTRAINT cns1 CHECK (c1 > 50), c2 text)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'check_constraints': {'cns1': { + 'columns': ['c1'], 'expression': '(c1 > 50)', + 'description': 'Test constraint cns1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "COMMENT ON CONSTRAINT cns1 ON s1.t1 IS " \ + "'Test constraint cns1'" diff --git a/tests/dbobject/test_conversion.py b/tests/dbobject/test_conversion.py new file mode 100644 index 0000000..303ee2b --- /dev/null +++ b/tests/dbobject/test_conversion.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +"""Test conversions""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE CONVERSION sd.c1 FOR 'LATIN1' TO 'UTF8' " \ + "FROM iso8859_1_to_utf8" +DROP_STMT = "DROP CONVERSION IF EXISTS sd.c1" +COMMENT_STMT = "COMMENT ON CONVERSION sd.c1 IS 'Test conversion c1'" + + +class ConversionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing conversions""" + + def test_map_conversion1(self): + "Map a conversion" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'} + assert dbmap['schema sd']['conversion c1'] == expmap + + def test_map_conversion_comment(self): + "Map a conversion comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['conversion c1']['description'] == \ + 'Test conversion c1' + + +class ConversionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input conversions""" + + def test_create_conversion(self): + "Create a conversion" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_conversion_schema(self): + "Create a conversion in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8', 'default': True}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE DEFAULT CONVERSION s1.c1 " \ + "FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8" + + def test_bad_conversion_map(self): + "Error creating a conversion with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_conversion(self): + "Drop an existing conversion" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql[0] == "DROP CONVERSION sd.c1" + + def test_conversion_with_comment(self): + "Create a conversion with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Test conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_conversion(self): + "Create a comment for an existing conversion" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Test conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_conversion_comment(self): + "Drop a comment on an existing conversion" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONVERSION sd.c1 IS NULL"] + + def test_change_conversion_comment(self): + "Change existing comment on a conversion" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Changed conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, stmts) + assert sql == [ + "COMMENT ON CONVERSION sd.c1 IS 'Changed conversion c1'"] diff --git a/tests/dbobject/test_domain.py b/tests/dbobject/test_domain.py new file mode 100644 index 0000000..35d3f7e --- /dev/null +++ b/tests/dbobject/test_domain.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""Test domains""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATEFUNC_STMT = ("CREATE FUNCTION sd.dc1(integer) RETURNS bool LANGUAGE sql " + "IMMUTABLE AS $_$select true$_$") +CREATE_STMT = "CREATE DOMAIN sd.d1 AS integer" +DROP_STMT = "DROP DOMAIN IF EXISTS d1" +COMMENT_STMT = "COMMENT ON DOMAIN d1 IS 'Test domain d1'" + + +class DomainToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created domains""" + + def test_domain(self): + "Map a simple domain" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['domain d1'] == {'type': 'integer'} + + def test_domain_not_null(self): + "Map a domain with a NOT NULL constraint" + dbmap = self.to_map([CREATE_STMT + " NOT NULL"]) + expmap = {'type': 'integer', 'not_null': True} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_default(self): + "Map a domain with a DEFAULT" + dbmap = self.to_map(["CREATE DOMAIN d1 AS date DEFAULT CURRENT_DATE"]) + expmap = {'type': 'date', 'default': 'CURRENT_DATE'} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_check(self): + "Map a domain with a CHECK constraint" + dbmap = self.to_map([CREATE_STMT + " CHECK (VALUE >= 1888)"]) + expmap = {'type': 'integer', 'check_constraints': { + 'd1_check': {'expression': '(VALUE >= 1888)'}}} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_depend_function(self): + "A domain is created after a function it depends on" + dbmap = self.to_map([CREATEFUNC_STMT, + CREATE_STMT + " CHECK (dc1(VALUE))"]) + expmap = {'type': 'integer', + 'check_constraints': { + 'd1_check': { + 'expression': 'sd.dc1(VALUE)', + 'depends_on': ['function dc1(integer)']}}} + assert dbmap['schema sd']['domain d1'] == expmap + + +class DomainToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input domains""" + + def test_create_domain_simple(self): + "Create a simple domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': {'type': 'integer'}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT] + + def test_create_domain_default(self): + "Create a domain with a DEFAULT and NOT NULL" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'not_null': True, 'default': 0}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT + " NOT NULL DEFAULT 0"] + + def test_create_domain_check(self): + "Create a domain with a CHECK constraint" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': {'d1_check': { + 'expression': '(VALUE >= 1888)'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert fix_indent(sql[1]) == "ALTER DOMAIN sd.d1 ADD CONSTRAINT " + \ + "d1_check CHECK (VALUE >= 1888)" + + def test_add_domain_check(self): + "Add a CHECK constraint to a domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': {'d1_check': { + 'expression': '(VALUE >= 1888)'}}}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "ALTER DOMAIN sd.d1 " + \ + "ADD CONSTRAINT d1_check CHECK (VALUE >= 1888)" + + def test_drop_domain(self): + "Drop an existing domain" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP DOMAIN sd.d1"] + + def test_rename_domain(self): + "Rename an existing domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d2': { + 'oldname': 'd1', 'type': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER DOMAIN sd.d1 RENAME TO d2"] + + def test_create_domain_before_function(self): + "Check that the domain is created after a function it depends on" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': { + 'd1_check': {'expression': 'dc1(VALUE)', + 'depends_on': ['function dc1(integer)']}}}, + 'function dc1(integer)': { + 'language': 'sql', 'returns': 'bool', + 'source': 'select true', 'volatility': 'immutable'}}) + sql = self.to_sql(inmap) + assert len(sql) == 4 + assert sql[1] == CREATE_STMT + assert fix_indent(sql[2]) == CREATEFUNC_STMT + assert sql[3].startswith("ALTER DOMAIN sd.d1 ADD CONSTRAINT") diff --git a/tests/dbobject/test_eventtrig.py b/tests/dbobject/test_eventtrig.py new file mode 100644 index 0000000..a666cb1 --- /dev/null +++ b/tests/dbobject/test_eventtrig.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +"""Test event triggers""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +FUNC_SRC = "BEGIN RAISE NOTICE 'Command % executed', tg_tag; END" +CREATE_FUNC_STMT = "CREATE FUNCTION sd.f1() RETURNS event_trigger " \ + "LANGUAGE plpgsql AS $_$%s$_$" % FUNC_SRC +CREATE_STMT = "CREATE EVENT TRIGGER et1 ON ddl_command_end %s" \ + "EXECUTE PROCEDURE sd.f1()" +DROP_TABLE_STMT = "DROP TABLE IF EXISTS t1" +DROP_FUNC_STMT = "DROP FUNCTION IF EXISTS f1()" +COMMENT_STMT = "COMMENT ON EVENT TRIGGER et1 IS 'Test event trigger et1'" + + +class EventTriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing event triggers""" + + def test_map_event_trigger_simple(self): + "Map a simple event trigger" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1'] == { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()'} + + def test_map_event_trigger_filter(self): + "Map a trigger with tag filter variables" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ( + "WHEN tag IN ('CREATE TABLE', 'CREATE VIEW') ")] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1'] == { + 'enabled': True, 'event': 'ddl_command_end', + 'tags': ['CREATE TABLE', 'CREATE VIEW'], 'procedure': 'sd.f1()'} + + def test_map_event_trigger_comment(self): + "Map a trigger comment" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % '', COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1']['description'] == \ + 'Test event trigger et1' + + +class EventTriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_event_trigger_simple(self): + "Create a simple event trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % '' + + def test_create_event_trigger_filter(self): + "Create an event trigger with tag filter variables" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()', 'tags': ['CREATE TABLE', 'CREATE VIEW']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % ( + "WHEN tag IN ('CREATE TABLE', 'CREATE VIEW') ") + + def test_create_event_trigger_func_schema(self): + "Create an event trigger with function in a non-default schema" + inmap = self.std_map(plpgsql_installed=True) + inmap.update({'schema s1': {'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 's1.f1()'}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE FUNCTION s1.f1() " \ + "RETURNS event_trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_SRC + assert fix_indent(sql[1]) == "CREATE EVENT TRIGGER et1 " \ + "ON ddl_command_end EXECUTE PROCEDURE s1.f1()" + + def test_drop_event_trigger(self): + "Drop an existing event trigger" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP EVENT TRIGGER et1"] + + def test_drop_event_trigger_function(self): + "Drop an existing event trigger and the related function" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + inmap = self.std_map(plpgsql_installed=True) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP EVENT TRIGGER et1" + assert sql[1] == "DROP FUNCTION sd.f1()" + + def test_create_event_trigger_with_comment(self): + "Create an event trigger with a comment" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()', 'description': 'Test event trigger et1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % '' + assert sql[2] == COMMENT_STMT diff --git a/tests/dbobject/test_extension.py b/tests/dbobject/test_extension.py new file mode 100644 index 0000000..d3a089f --- /dev/null +++ b/tests/dbobject/test_extension.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +"""Test extensions""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE EXTENSION pg_trgm" +TRGM_COMMENT = "text similarity measurement and index searching based on " \ + "trigrams" +PLR_DESCR = "load R interpreter and execute R script from within a database" + + +class ExtensionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing extensions""" + + superuser = True + def base_version(self): + if self.db.version < 110000: + return '1.3' + elif self.db.version < 120000: + return '1.4' + elif self.db.version < 150000: + return '1.5' + return '1.6' + + def test_map_extension(self): + "Map an existing extension" + VERS = self.base_version() + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['extension pg_trgm'] == { + 'schema': 'sd', 'version': VERS, 'description': TRGM_COMMENT} + + def test_map_no_depends(self): + "Ensure no dependencies are included when mapping an extension" + dbmap = self.to_map([CREATE_STMT]) + assert 'type gtrgm' not in dbmap['schema sd'] + assert 'operator %(text, text)' not in dbmap['schema sd'] + assert 'function show_trgm(text)' not in dbmap['schema sd'] + + def test_map_lang_extension(self): + "Map a procedural language as an extension" + dbmap = self.to_map(["CREATE EXTENSION plperl"]) + assert dbmap['extension plperl'] == { + 'schema': 'pg_catalog', 'version': '1.0', + 'description': "PL/Perl procedural language"} + assert 'language plperl' not in dbmap + + def test_map_extern_lang_extension(self): + "Map an externally maintained procedural language" + if self.db.version < 150000: + self.skipTest('Only available since PG 15') + dbmap = self.to_map(["CREATE EXTENSION plr"]) + assert dbmap['extension plr'] == { + 'schema': 'sd', 'version': '8.4.5', 'description': PLR_DESCR} + assert dbmap['language plr'] == {'trusted': False} + # Ensure we don't map core-maintained language + assert 'language plpgsql' not in dbmap + + def test_map_extension_schema(self): + "Map an existing extension" + VERS = self.base_version() + dbmap = self.to_map(["CREATE SCHEMA s1", CREATE_STMT + " SCHEMA s1"]) + assert dbmap['extension pg_trgm'] == { + 'schema': 's1', 'version': VERS, 'description': TRGM_COMMENT} + + def test_map_extension_plpython3u(self): + "Test a function created with extension other than plpgsql/plperl" + # See issue #103 + dbmap = self.to_map(["CREATE EXTENSION plpython3u", + "CREATE FUNCTION test() RETURNS int AS " + "'return 1' LANGUAGE plpython3u"]) + assert 'extension plpython3u' in dbmap + + +class ExtensionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input extensions""" + + def test_create_extension_simple(self): + "Create a extension that didn't exist" + inmap = self.std_map() + inmap.update({'extension pg_trgm': {'schema': 'sd'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + " SCHEMA sd" + + def test_bad_extension_map(self): + "Error creating a extension with a bad map" + inmap = self.std_map() + inmap.update({'pg_trgm': {'schema': 'sd'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_extension(self): + "Drop an existing extension" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP EXTENSION pg_trgm"] + + def test_create_extension_schema(self): + "Create a extension in a given schema" + inmap = self.std_map() + inmap.update({'schema s1': {}, + 'extension pg_trgm': {'schema': 's1', 'version': '1.0'}}) + sql = self.to_sql(inmap) + assert sql[0] == 'CREATE SCHEMA s1' + assert fix_indent(sql[1]) == \ + "CREATE EXTENSION pg_trgm SCHEMA s1 VERSION '1.0'" + + def test_create_lang_extension(self): + "Create a language extension and a function in that language" + inmap = self.std_map() + inmap.update({'extension plperl': {'schema': 'pg_catalog', + 'description': + "PL/Perl procedural language"}}) + inmap['schema sd'].update({'function f1()': { + 'language': 'plperl', 'returns': 'text', + 'source': "return \"dummy\";"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE EXTENSION plperl" + # skip over COMMENT statement + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE plperl AS $_$return \"dummy\";$_$" + + def test_create_extern_lang_extension(self): + "Create an externally maintained language together with a function" + inmap = self.std_map() + inmap.update({'extension plr': {'schema': 'sd', + 'description': PLR_DESCR}}) + inmap.update({'language plr': { 'trusted': 'false'}}) + inmap['schema sd'].update({'function f1()': { + 'language': 'plr', 'returns': 'text', + 'source': "return \"dummy\";"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE EXTENSION plr SCHEMA sd" + # skip over COMMENT statement + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE plr AS $_$return \"dummy\";$_$" + + def test_comment_extension(self): + "Change the comment for an existing extension" + inmap = self.std_map() + inmap.update({'extension pg_trgm': { + 'schema': 'sd', 'description': "Trigram extension"}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == ["COMMENT ON EXTENSION pg_trgm IS 'Trigram extension'"] + + def test_no_alter_owner_extension(self): + """Do not alter the owner of an existing extension. + + ALTER EXTENSION extension_name OWNER is not a valid form. + """ + # create a new owner that is different from self.db.user + new_owner = 'new_%s' % self.db.user + inmap = self.std_map() + inmap.update({'extension pg_trgm': {'schema': 'sd', + 'owner': new_owner}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert 'ALTER EXTENSION pg_trgm OWNER TO %s' % new_owner not in sql diff --git a/tests/dbobject/test_extern_file.py b/tests/dbobject/test_extern_file.py new file mode 100644 index 0000000..0e114f6 --- /dev/null +++ b/tests/dbobject/test_extern_file.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Test external files used in --multiple-files option""" +import sys + +import pytest + +from pyrseas.testutils import PyrseasTestCase +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.dbobject.schema import Schema +from pyrseas.dbobject.function import Function +from pyrseas.dbobject.table import Sequence, Table +from pyrseas.dbobject.view import View + +if sys.platform == 'win32': + COLL = 'French_France.1252' +else: + COLL = 'fr_FR.UTF-8' + +CREATE_FDW = "CREATE FOREIGN DATA WRAPPER " +SOURCE1 = "SELECT 'dummy'::text" +SOURCE2 = "SELECT $1::text" +DROP_TSC = "DROP TEXT SEARCH CONFIGURATION IF EXISTS tsc1, tsc2" +DROP_TSP = "DROP TEXT SEARCH PARSER IF EXISTS tsp1 CASCADE" + + +class ExternalFilenameMapTestCase(DatabaseToMapTestCase): + + def setUp(self): + super(ExternalFilenameMapTestCase, self).setUp() + self.remove_tempfiles() + + def test_map_casts(self): + "Map casts" + self.to_map(["CREATE FUNCTION int2_bool(smallint) RETURNS boolean " + "LANGUAGE sql IMMUTABLE AS " + "$_$SELECT CAST($1::int AS boolean)$_$", + "CREATE DOMAIN d1 AS integer", + "CREATE CAST (smallint AS boolean) WITH FUNCTION " + "int2_bool(smallint)", + "CREATE CAST (d1 AS integer) WITH INOUT AS IMPLICIT"], + superuser=True, multiple_files=True) + expmap = {'cast (smallint as boolean)': { + 'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}, 'cast (sd.d1 as integer)': + {'context': 'implicit', 'method': 'inout', + 'depends_on': ['domain d1']}} + assert self.yaml_load('cast.yaml') == expmap + + def test_map_extension(self): + "Map extensions" + TRGM_VERS = '1.6' + if self.db.version < 110000: + TRGM_VERS = '1.3' + elif self.db.version < 130000: + TRGM_VERS = '1.4' + elif self.db.version < 150000: + TRGM_VERS = '1.5' + self.to_map(["CREATE EXTENSION pg_trgm"], superuser=True, + multiple_files=True) + expmap = {'extension plpgsql': { + 'schema': 'pg_catalog', 'version': '1.0', + 'description': 'PL/pgSQL procedural language'}, + 'extension pg_trgm': {'schema': 'sd', 'version': TRGM_VERS, + 'description': "text similarity measurement " + "and index searching based on trigrams"}} + assert self.yaml_load('extension.yaml') == expmap + + def test_map_fd_wrappers(self): + "Map foreign data wrappers" + self.to_map([CREATE_FDW + "fdw1", CREATE_FDW + "fdw2 " + "OPTIONS (debug 'true')"], superuser=True, + multiple_files=True) + expmap = {'foreign data wrapper fdw1': {}, + 'foreign data wrapper fdw2': {'options': ['debug=true']}} + assert self.yaml_load('foreign_data_wrapper.yaml') == expmap + + def test_collations(self): + "Map collations" + self.to_map(["CREATE COLLATION coll1 (LC_COLLATE = '%s', " + "LC_CTYPE = '%s')" % (COLL, COLL), + "COMMENT ON COLLATION coll1 IS 'A test collation'", + "CREATE COLLATION coll2 (LC_COLLATE = '%s', " + "LC_CTYPE = '%s')" % (COLL, COLL)], + multiple_files=True) + assert self.yaml_load('collation.yaml', 'schema.sd') == { + 'collation coll1': {'lc_collate': COLL, 'lc_ctype': COLL, + 'description': "A test collation"}, + 'collation coll2': {'lc_collate': COLL, 'lc_ctype': COLL}} + + def test_map_conversion(self): + "Map conversions" + self.to_map(["CREATE CONVERSION conv1 FOR 'LATIN1' TO 'UTF8' " + "FROM iso8859_1_to_utf8", + "COMMENT ON CONVERSION conv1 IS 'A test conversion'"], + multiple_files=True) + assert self.yaml_load('conversion.yaml', 'schema.sd') == { + 'conversion conv1': {'source_encoding': 'LATIN1', + 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8', + 'description': 'A test conversion'}} + + def test_map_functions(self): + "Map functions" + self.to_map(["CREATE FUNCTION f1() RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE1, + "CREATE FUNCTION f2() RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE1], multiple_files=True) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable'} + assert self.yaml_load('function.f1.yaml', 'schema.sd') == { + 'function f1()': expmap} + assert self.yaml_load('function.f2.yaml', 'schema.sd') == { + 'function f2()': expmap} + + def test_map_functions_merged(self): + "Map functions into a merged file" + self.to_map(["CREATE FUNCTION f3(integer) RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE2, + "CREATE FUNCTION f3(real) RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE2], multiple_files=True) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE2, 'volatility': 'immutable'} + assert self.yaml_load('function.f3.yaml', 'schema.sd') == { + 'function f3(integer)': expmap, 'function f3(real)': expmap} + + def test_map_operator(self): + "Map operators and an operator class" + self.to_map(["CREATE OPERATOR < (PROCEDURE = int4lt, LEFTARG = int, " + "RIGHTARG = int)", "CREATE OPERATOR = (PROCEDURE = " + "int4eq, LEFTARG = int, RIGHTARG = int)", "CREATE " + "OPERATOR > (PROCEDURE = int4gt, LEFTARG = int, " + "RIGHTARG = int)", + "CREATE OPERATOR CLASS oc1 FOR TYPE integer USING btree " + "AS OPERATOR 1 sd.<, OPERATOR 3 sd.=, OPERATOR " + "5 sd.>, FUNCTION 1 btint4cmp(integer,integer)"], + superuser=True, multiple_files=True) + oprmap = {'operator <(integer, integer)': {'procedure': 'int4lt'}, + 'operator =(integer, integer)': {'procedure': 'int4eq'}, + 'operator >(integer, integer)': {'procedure': 'int4gt'}} + opcmap = {'operator class oc1 using btree': { + 'type': 'integer', 'operators': { + 1: 'sd.<(integer,integer)', 3: 'sd.=(integer,integer)', + 5: 'sd.>(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}} + assert self.yaml_load('operator.yaml', 'schema.sd') == oprmap + assert self.yaml_load('operator_class.yaml', 'schema.sd') == opcmap + + def test_map_operator_family(self): + "Map operator families" + self.to_map(["CREATE SCHEMA s1", + "CREATE OPERATOR FAMILY s1.of1 USING btree", + "CREATE OPERATOR FAMILY s1.of2 USING btree"], + superuser=True, multiple_files=True) + assert self.yaml_load('operator_family.yaml', 'schema.s1') == { + 'operator family of1 using btree': {}, + 'operator family of2 using btree': {}} + + def test_map_tables(self): + "Map tables" + self.to_map(["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CREATE TABLE t2 (c1 integer, c2 integer, c3 text, " + "FOREIGN KEY (c2) REFERENCES t1 (c1))"], + multiple_files=True) + expmap1 = {'table t1': {'columns': [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}} + expmap2 = {'table t2': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t2_c2_fkey': {'columns': ['c2'], 'references': { + 'schema': 'sd', 'table': 't1', 'columns': ['c1']}}}}} + assert self.yaml_load('table.t1.yaml', 'schema.sd') == expmap1 + assert self.yaml_load('table.t2.yaml', 'schema.sd') == expmap2 + + def test_map_tables_merged(self): + "Map tables into a merged file" + self.to_map(["CREATE TABLE account_transfers_with_extra_padding_1 " + "(c1 integer, c2 text)", + "CREATE TABLE account_transfers_with_extra_padding_2 " + "(c1 integer, c2 text)"], + multiple_files=True) + expmap = {'table account_transfers_with_extra_padding_1': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]}, + 'table account_transfers_with_extra_padding_2': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]}} + assert self.yaml_load('table.account_transfers_with_extra_pad.yaml', + 'schema.sd') == expmap + + def test_map_textsearch(self): + "Map text search components" + self.to_map([DROP_TSC, DROP_TSP, + "CREATE TEXT SEARCH PARSER tsp1 (START = prsd_start, " + "GETTOKEN = prsd_nexttoken, END = prsd_end, " + "LEXTYPES = prsd_lextype, HEADLINE = prsd_headline)", + "CREATE TEXT SEARCH CONFIGURATION tsc1 (PARSER = tsp1)", + "CREATE TEXT SEARCH CONFIGURATION tsc2 (PARSER = tsp1)"], + superuser=True, multiple_files=True) + assert self.yaml_load('text_search_configuration.yaml', + 'schema.sd') == \ + {'text search configuration tsc1': {'parser': 'tsp1'}, + 'text search configuration tsc2': {'parser': 'tsp1'}} + assert self.yaml_load('text_search_parser.yaml', 'schema.sd') == { + 'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}} + self.db.execute(DROP_TSC) + self.db.execute_commit(DROP_TSP) + + def test_map_drop_table(self): + "Map three tables, drop one and map the remaining two" + self.to_map(["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE TABLE t3 (c1 integer, c2 text)"], + multiple_files=True) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + for tbl in ['t1', 't2', 't3']: + assert self.yaml_load('table.%s.yaml' % tbl, 'schema.sd')[ + 'table %s' % tbl] == expmap + self.to_map(["DROP TABLE t2"], multiple_files=True) + with pytest.raises(IOError) as exc: + self.yaml_load('table.t2.yaml', 'schema.sd') + assert 'No such file' in str(exc.value) + for tbl in ['t1', 't3']: + assert self.yaml_load('table.%s.yaml' % tbl, 'schema.sd')[ + 'table %s' % tbl] == expmap + + +class ExternalFilenameTestCase(PyrseasTestCase): + + def test_function(self): + "Map a function" + obj = Function("Weird/Or-what?", 'sd', '', None, [], None, None, + None, None) + assert obj.extern_filename() == 'function.weird_or_what_.yaml' + + def test_schema(self): + "Map a schema" + obj = Schema(name="A/C Schema") + assert obj.extern_filename() == 'schema.a_c_schema.yaml' + + def test_long_name_schema(self): + "Map a schema with a long name" + nm = 'a_schema_with_a_very_but_very_very_long_long_long_loooonng_name' + obj = Schema(name=nm) + assert obj.extern_filename() == 'schema.%s.yaml' % nm + + def test_table(self): + "Map a table" + obj = Table("Weird/Or-what?HOW.WeiRD", '', None, None, []) + assert obj.extern_filename() == 'table.weird_or_what_how_weird.yaml' + + def test_table_unicode(self): + "Map a table with Unicode characters" + obj = Table("Fundação\\Größe таблица", '', None, None, []) + assert obj.extern_filename() == 'table.fundação_größe_таблица.yaml' + + def test_sequence(self): + "Map a sequence" + obj = Sequence("Weird/Or-what?_seq", '', None, None, []) + assert obj.extern_filename() == 'sequence.weird_or_what__seq.yaml' + + def test_view(self): + "Map a view" + obj = View("Weirder/Don't You Think?", '', None, None, [], '') + assert obj.extern_filename() == 'view.weirder_don_t_you_think_.yaml' diff --git a/tests/dbobject/test_foreign.py b/tests/dbobject/test_foreign.py new file mode 100644 index 0000000..ce9634c --- /dev/null +++ b/tests/dbobject/test_foreign.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +"""Test text search objects""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_FDW_STMT = "CREATE FOREIGN DATA WRAPPER fdw1" +CREATE_FS_STMT = "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1" +CREATE_UM_STMT = "CREATE USER MAPPING FOR PUBLIC SERVER fs1" +CREATE_FT_STMT = "CREATE FOREIGN TABLE sd.ft1 (c1 integer, c2 text) SERVER fs1" +DROP_FDW_STMT = "DROP FOREIGN DATA WRAPPER IF EXISTS fdw1" +DROP_FS_STMT = "DROP SERVER IF EXISTS fs1" +DROP_UM_STMT = "DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER fs1" +COMMENT_FDW_STMT = "COMMENT ON FOREIGN DATA WRAPPER fdw1 IS " \ + "'Test foreign data wrapper fdw1'" +COMMENT_FS_STMT = "COMMENT ON SERVER fs1 IS 'Test server fs1'" + + +class ForeignDataWrapperToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign data wrappers""" + + superuser = True + + def test_map_fd_wrapper(self): + "Map an existing foreign data wrapper" + dbmap = self.to_map([CREATE_FDW_STMT]) + assert dbmap['foreign data wrapper fdw1'] == {} + + def test_map_wrapper_validator(self): + "Map a foreign data wrapper with a validator function" + dbmap = self.to_map(["CREATE FOREIGN DATA WRAPPER fdw1 " + "VALIDATOR postgresql_fdw_validator"]) + assert dbmap['foreign data wrapper fdw1'] == { + 'validator': 'postgresql_fdw_validator'} + + def test_map_wrapper_options(self): + "Map a foreign data wrapper with options" + dbmap = self.to_map(["CREATE FOREIGN DATA WRAPPER fdw1 " + "OPTIONS (debug 'true')"]) + assert dbmap['foreign data wrapper fdw1'] == { + 'options': ['debug=true']} + + def test_map_fd_wrapper_comment(self): + "Map a foreign data wrapper with a comment" + dbmap = self.to_map([CREATE_FDW_STMT, COMMENT_FDW_STMT]) + assert dbmap['foreign data wrapper fdw1']['description'] == \ + 'Test foreign data wrapper fdw1' + + +class ForeignDataWrapperToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign data wrappers""" + + def test_create_fd_wrapper(self): + "Create a foreign data wrapper that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + + def test_create_wrapper_validator(self): + "Create a foreign data wrapper with a validator function" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'validator': 'postgresql_fdw_validator'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FOREIGN DATA WRAPPER fdw1 " \ + "VALIDATOR postgresql_fdw_validator" + + def test_create_wrapper_options(self): + "Create a foreign data wrapper with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'options': ['debug=true']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FOREIGN DATA WRAPPER fdw1 " \ + "OPTIONS (debug 'true')" + + def test_bad_map_fd_wrapper(self): + "Error creating a foreign data wrapper with a bad map" + inmap = self.std_map() + inmap.update({'fdw1': {}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_fd_wrapper(self): + "Drop an existing foreign data wrapper" + sql = self.to_sql(self.std_map(), [CREATE_FDW_STMT], superuser=True) + assert sql[0] == "DROP FOREIGN DATA WRAPPER fdw1" + + def test_alter_wrapper_options(self): + "Change foreign data wrapper options" + stmts = [CREATE_FDW_STMT + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'options': ['opt1=valX', 'opt3=valY']}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER FOREIGN DATA WRAPPER fdw1 " + assert sorted(sql[1][1:-1].split(', ')) == [ + "DROP opt2", "SET opt1 'valX'", "opt3 'valY'"] + + def test_comment_on_fd_wrapper(self): + "Create a comment for an existing foreign data wrapper" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'description': "Test foreign data wrapper fdw1"}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert sql[0] == COMMENT_FDW_STMT + + +class ForeignServerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign servers""" + + superuser = True + + def test_map_server(self): + "Map an existing foreign server" + dbmap = self.to_map([CREATE_FDW_STMT, CREATE_FS_STMT]) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': {}} + + def test_map_server_type_version(self): + "Map a foreign server with type and version" + stmts = [CREATE_FDW_STMT, + "CREATE SERVER fs1 TYPE 'test' VERSION '1.0' " + "FOREIGN DATA WRAPPER fdw1"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'type': 'test', 'version': '1.0'}} + + def test_map_server_options(self): + "Map a foreign server with options" + stmts = [CREATE_FDW_STMT, + "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1 " + "OPTIONS (dbname 'test')"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'options': ['dbname=test']}} + + def test_map_server_comment(self): + "Map a foreign server with a comment" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, COMMENT_FS_STMT] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'description': 'Test server fs1'}} + + +class ForeignServerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign servers""" + + def test_create_server(self): + "Create a foreign server that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == CREATE_FS_STMT + + def test_create_wrapper_server(self): + "Create a foreign data wrapper and its server" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + assert fix_indent(sql[1]) == CREATE_FS_STMT + + def test_create_server_type_version(self): + "Create a foreign server with type and version" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'type': 'test', 'version': '1.0'}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE SERVER fs1 TYPE 'test' " \ + "VERSION '1.0' FOREIGN DATA WRAPPER fdw1" + + def test_create_server_options(self): + "Create a foreign server with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'options': ['dbname=test']}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE SERVER fs1 " \ + "FOREIGN DATA WRAPPER fdw1 OPTIONS (dbname 'test')" + + def test_drop_server(self): + "Drop an existing foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql[0] == "DROP SERVER fs1" + + def test_add_server_options(self): + "Add options to a foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'options': ['opt1=valA', 'opt2=valB']}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER SERVER fs1 " + assert sorted(sql[1][1:-1].split(', ')) == [ + "opt1 'valA'", "opt2 'valB'"] + + def test_drop_server_wrapper(self): + "Drop an existing foreign data wrapper and its server" + sql = self.to_sql(self.std_map(), [CREATE_FDW_STMT, CREATE_FS_STMT], + superuser=True) + assert sql[0] == "DROP SERVER fs1" + assert sql[1] == "DROP FOREIGN DATA WRAPPER fdw1" + + def test_comment_on_server(self): + "Create a comment for an existing foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'description': "Test server fs1"}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_FS_STMT] + + +class UserMappingToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing user mappings""" + + superuser = True + + def test_map_user_mapping(self): + "Map an existing user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1']['server fs1'][ + 'user mappings'] == {'PUBLIC': {}} + + def test_map_user_mapping_options(self): + "Map a user mapping with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT + + " OPTIONS (user 'john', password 'doe')"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1']['server fs1'] == { + 'user mappings': {'PUBLIC': {'options': [ + 'password=doe', 'user=john']}}} + + +class UserMappingToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input user mappings""" + + def test_create_user_mapping(self): + "Create a user mapping that didn't exist" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert fix_indent(sql[0]) == CREATE_UM_STMT + + def test_create_wrapper_server_mapping(self): + "Create a FDW, server and user mapping" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + assert fix_indent(sql[1]) == CREATE_FS_STMT + assert fix_indent(sql[2]) == CREATE_UM_STMT + + def test_create_user_mapping_options(self): + "Create a user mapping with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {'options': [ + 'user=john', 'password=doe']}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert fix_indent(sql[0]) == CREATE_UM_STMT + \ + " OPTIONS (user 'john', password 'doe')" + + def test_drop_user_mapping(self): + "Drop an existing user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP USER MAPPING FOR PUBLIC SERVER fs1" + + def test_drop_user_mapping_options(self): + "Drop options from a user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, + CREATE_UM_STMT + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER USER MAPPING FOR PUBLIC SERVER fs1 " + assert sorted(sql[1][1:-1].split(', ')) == ["DROP opt1", "DROP opt2"] + + +class ForeignTableToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign tables""" + + superuser = True + + def test_map_foreign_table(self): + "Map an existing foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1'} + assert dbmap['schema sd']['foreign table ft1'] == expmap + + def test_map_foreign_table_options(self): + "Map a foreign table with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT + + " OPTIONS (user 'jack')"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'options': ['user=jack']} + assert dbmap['schema sd']['foreign table ft1'] == expmap + + +class ForeignTableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign tables""" + + superuser = True + + def test_create_foreign_table(self): + "Create a foreign table that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1'}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT, CREATE_FS_STMT]) + assert fix_indent(sql[0]) == CREATE_FT_STMT + + def test_create_foreign_table_options(self): + "Create a foreign table with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'options': ['user=jack']}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT, CREATE_FS_STMT]) + assert fix_indent(sql[0]), CREATE_FT_STMT + " OPTIONS (user 'jack')" + + def test_bad_map_foreign_table(self): + "Error creating a foreign table with a bad map" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_foreign_table(self): + "Drop an existing foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP FOREIGN TABLE sd.ft1" + + def test_drop_foreign_table_server(self): + "Drop a foreign table and associated server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP FOREIGN TABLE sd.ft1" + assert sql[1] == "DROP SERVER fs1" + + def test_alter_foreign_table_options(self): + "Alter options for a foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT + + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'options': ['opt1=valX', 'opt2=valY']}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]), "ALTER FOREIGN TABLE ft1 " \ + "OPTIONS (SET opt1 'valX', SET opt2 'valY')" + + def test_add_column(self): + "Add new column to a foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], 'server': 'fs1'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER FOREIGN TABLE sd.ft1 ADD COLUMN c3 date" diff --git a/tests/dbobject/test_function.py b/tests/dbobject/test_function.py new file mode 100644 index 0000000..42d0446 --- /dev/null +++ b/tests/dbobject/test_function.py @@ -0,0 +1,522 @@ +# -*- coding: utf-8 -*- +"""Test functions""" + +import pytest + +from inspect import cleandoc + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +SOURCE1 = "SELECT 'dummy'::text" +CREATE_STMT1 = "CREATE FUNCTION sd.f1() RETURNS text LANGUAGE sql IMMUTABLE " \ + "AS $_$%s$_$" % SOURCE1 + +SOURCE2 = "SELECT GREATEST($1, $2)" +CREATE_STMT2 = "CREATE FUNCTION sd.f1(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE2 +COMMENT_STMT = "COMMENT ON FUNCTION sd.f1(integer, integer) IS " \ + "'Test function f1'" + +SOURCE3 = "SELECT * FROM generate_series($1, $2)" +CREATE_STMT3 = "CREATE FUNCTION f2(integer, integer) RETURNS SETOF integer " \ + "ROWS 20 LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE3 + +SOURCE4 = "SELECT $1 + $2" +CREATE_STMT4 = "CREATE FUNCTION fadd(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE4 + +SOURCE5 = "SELECT $1 - $2" +CREATE_STMT5 = "CREATE FUNCTION fsub(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE5 + + +class FunctionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing functions""" + + def test_map_function1(self): + "Map a very simple function with no arguments" + dbmap = self.to_map([CREATE_STMT1]) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable'} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_function_with_args(self): + "Map a function with two arguments" + stmts = ["CREATE FUNCTION f1(integer, integer) RETURNS integer " + "LANGUAGE sql AS $_$%s$_$" % SOURCE2] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2} + + def test_map_function_default_args(self): + "Map a function with default arguments" + stmts = ["CREATE FUNCTION f1(integer, integer) RETURNS integer " + "LANGUAGE sql AS $_$%s$_$" % SOURCE2] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2} + + def test_map_void_function(self): + "Map a function returning void" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE FUNCTION f1() RETURNS void LANGUAGE sql AS " + "$_$INSERT INTO t1 VALUES (1, 'dummy')$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'void', + 'source': "INSERT INTO t1 VALUES (1, 'dummy')"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_setof_row_function(self): + "Map a function returning a set of rows" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE FUNCTION f1() RETURNS SETOF t1 LANGUAGE sql AS " + "$_$SELECT * FROM t1$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'SETOF sd.t1', + 'source': "SELECT * FROM t1"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_security_definer_function(self): + "Map a function that is SECURITY DEFINER" + stmts = ["CREATE FUNCTION f1() RETURNS text LANGUAGE sql " + "SECURITY DEFINER AS $_$%s$_$" % SOURCE1] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'security_definer': True} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_c_lang_function(self): + "Map a dynamically loaded C language function" + # NOTE 1: Needs contrib/spi module to be available + # NOTE 2: Needs superuser privilege + stmts = ["CREATE FUNCTION autoinc() RETURNS trigger " + "AS '$libdir/autoinc' LANGUAGE c"] + dbmap = self.to_map(stmts, superuser=True) + expmap = {'language': 'c', 'obj_file': '$libdir/autoinc', + 'link_symbol': 'autoinc', 'returns': 'trigger'} + assert dbmap['schema sd']['function autoinc()'] == expmap + + def test_map_function_config(self): + "Map a function with a configuration parameter" + stmts = ["CREATE FUNCTION f1() RETURNS date LANGUAGE sql SET " + "datestyle to postgres, dmy AS $_$SELECT CURRENT_DATE$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'date', + 'configuration': ['DateStyle=postgres, dmy'], + 'source': "SELECT CURRENT_DATE"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_function_comment(self): + "Map a function comment" + dbmap = self.to_map([CREATE_STMT2, COMMENT_STMT]) + assert dbmap['schema sd']['function f1(integer, integer)'][ + 'description'] == 'Test function f1' + + def test_map_function_rows(self): + "Map a function rows" + dbmap = self.to_map([CREATE_STMT3]) + assert dbmap['schema sd']['function f2(integer, integer)'][ + 'rows'] == 20 + + def test_map_function_leakproof(self): + "Map a function with LEAKPROOF qualifier" + stmt = CREATE_STMT4.replace("IMMUTABLE", "IMMUTABLE LEAKPROOF") + dbmap = self.to_map([stmt], superuser=True) + expmap = {'language': 'sql', 'returns': 'integer', 'leakproof': True, + 'source': SOURCE4, 'volatility': 'immutable'} + assert dbmap['schema sd']['function fadd(integer, integer)'] == \ + expmap + + +class FunctionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input functions""" + + def test_create_function1(self): + "Create a very simple function with no arguments" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == CREATE_STMT1 + + def test_create_function_with_args(self): + "Create a function with two arguments" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + + def test_create_setof_row_function(self): + "Create a function returning a set of rows" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({ + 'function f1()': {'language': 'sql', 'returns': 'SETOF t1', + 'source': "SELECT * FROM t1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "SETOF t1 LANGUAGE sql AS $_$SELECT * FROM t1$_$" + + def test_create_setof_row_function_rows(self): + "Create a function returning a set of rows with suggested number" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({ + 'function f1()': {'language': 'sql', 'returns': 'SETOF t1', + 'source': "SELECT * FROM t1", 'rows': 50}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS SETOF " \ + "t1 LANGUAGE sql ROWS 50 AS $_$SELECT * FROM t1$_$" + + def test_create_security_definer_function(self): + "Create a SECURITY DEFINER function" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'security_definer': True}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE sql SECURITY DEFINER AS $_$%s$_$" % SOURCE1 + + def test_create_c_lang_function(self): + "Create a dynamically loaded C language function" + # NOTE 1: Needs contrib/spi module to be available + # NOTE 2: Needs superuser privilege + inmap = self.std_map() + inmap['schema sd'].update({'function autoinc()': { + 'language': 'c', 'returns': 'trigger', + 'obj_file': '$libdir/autoinc'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FUNCTION sd.autoinc() " \ + "RETURNS trigger LANGUAGE c AS '$libdir/autoinc', 'autoinc'" + + def test_create_function_config(self): + "Create a function with a configuration parameter" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'date', + 'configuration': ['DateStyle=postgres, dmy'], + 'source': "SELECT CURRENT_DATE"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS date " \ + "LANGUAGE sql SET DateStyle=postgres, dmy AS " \ + "$_$SELECT CURRENT_DATE$_$" + + def test_create_function_in_schema(self): + "Create a function within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[1]) == "CREATE FUNCTION s1.f1() RETURNS text " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE1 + + def test_bad_function_map(self): + "Error creating a function with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_function1(self): + "Drop an existing function with no arguments" + sql = self.to_sql(self.std_map(), [CREATE_STMT1]) + assert sql == ["DROP FUNCTION sd.f1()"] + + def test_drop_function_with_args(self): + "Drop an existing function which has arguments" + sql = self.to_sql(self.std_map(), [CREATE_STMT2]) + assert sql == ["DROP FUNCTION sd.f1(integer, integer)"] + + def test_change_function_defn(self): + "Change function definition" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', + 'source': "SELECT 'example'::text", 'volatility': 'immutable'}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[1]) == "CREATE OR REPLACE FUNCTION sd.f1() " \ + "RETURNS text LANGUAGE sql IMMUTABLE AS " \ + "$_$SELECT 'example'::text$_$" + + def test_function_with_comment(self): + "Create a function with a comment" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Test function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + assert sql[2] == COMMENT_STMT + + def test_comment_on_function(self): + "Create a comment for an existing function" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Test function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert sql == [COMMENT_STMT] + + def test_drop_function_comment(self): + "Drop a comment on an existing function" + stmts = [CREATE_STMT2, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON FUNCTION sd.f1(integer, integer) IS NULL"] + + def test_change_function_comment(self): + "Change existing comment on a function" + stmts = [CREATE_STMT2, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Changed function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON FUNCTION sd.f1(integer, integer) IS " + "'Changed function f1'"] + + def test_function_leakproof(self): + "Create a function with LEAKPROOF qualifier" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'leakproof': True, + 'source': SOURCE4, 'volatility': 'immutable'}}) + sql = self.to_sql(inmap, superuser=True) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql IMMUTABLE LEAKPROOF AS " \ + "$_$%s$_$" % SOURCE4 + + def test_alter_function_leakproof(self): + "Change a function with LEAKPROOF qualifier" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function fadd(integer, integer)': { + 'language': 'sql', 'returns': 'integer', + 'source': SOURCE4, 'volatility': 'immutable'}}) + stmt = CREATE_STMT4.replace("IMMUTABLE", "IMMUTABLE LEAKPROOF") + sql = self.to_sql(inmap, [stmt], superuser=True) + assert fix_indent(sql[0]) == \ + "ALTER FUNCTION sd.fadd(integer, integer) NOT LEAKPROOF" + + def test_change_function_return_type(self): + source = lambda rtype: "SELECT '127.0.0.1'::{}".format(rtype) + old_type = 'text' + new_type = 'inet' + statement = lambda rtype: cleandoc(""" + CREATE OR REPLACE FUNCTION sd.fget_addr() + RETURNS {rtype} + LANGUAGE sql + IMMUTABLE + AS $_${body}$_$""" + ).format( + rtype=rtype, + body=source(rtype), + ).replace('\n', ' ') + + inmap = self.std_map() + inmap['schema sd'].update({ + 'function fget_addr()': { + 'language': 'sql', + 'returns': new_type, + 'source': source(new_type), + } + }) + sql = self.to_sql(inmap, [statement(old_type)]) + assert statement(new_type) == fix_indent(sql[1]) + + +class AggregateToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing aggregates""" + + def test_map_aggregate_simple(self): + "Map a simple aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer'} + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_aggregate_init_final(self): + "Map an aggregate with an INITCOND and a FINALFUNC" + stmts = [CREATE_STMT2, + "CREATE FUNCTION f2(integer) RETURNS float " + "LANGUAGE sql AS $_$SELECT $1::float$_$ IMMUTABLE", + "CREATE AGGREGATE a1 (integer) (SFUNC = f1, STYPE = integer, " + "FINALFUNC = f2, INITCOND = '-1')"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', + 'initcond': '-1', 'finalfunc': 'f2'} + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'} + assert dbmap['schema sd']['function f2(integer)'] == \ + {'language': 'sql', 'returns': 'double precision', + 'source': "SELECT $1::float", 'volatility': 'immutable'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_aggregate_sortop(self): + "Map an aggregate with a SORTOP" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer, SORTOP = >)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', + 'sortop': 'pg_catalog.>'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_moving_aggregate(self): + "Map a moving-aggregate mode function" + stmts = [CREATE_STMT4, CREATE_STMT5, + "CREATE AGGREGATE a1 (integer) (sfunc = fadd, " + "stype = integer, initcond = '0', msfunc = fadd, " + "minvfunc = fsub, mstype = integer, minitcond = '0')"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'fadd', 'stype': 'integer', 'initcond': '0', + 'msfunc': 'fadd', 'minvfunc': 'fsub', 'mstype': 'integer', + 'minitcond': '0'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_ordered_set_aggregate(self): + "Map an ordered-set aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer ORDER BY " + "integer) (sfunc = f1, stype = integer)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', 'kind': 'ordered'} + assert dbmap['schema sd'][ + 'aggregate a1(integer ORDER BY integer)'] == expmap + + def test_map_aggregate_restricted(self): + "Map an aggregate with restricted parallel safety" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer, PARALLEL = RESTRICTED)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', 'parallel': 'restricted'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + +class AggregateToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input aggregates""" + + def test_create_aggregate_simple(self): + "Create a simple aggregate" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(integer)': { + 'sfunc': 'f1', 'stype': 'integer'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == CREATE_STMT2 + assert fix_indent(sql[2]) == "CREATE AGGREGATE sd.a1(integer) " \ + "(SFUNC = sd.f1, STYPE = integer)" + + def test_create_aggregate_sortop(self): + "Create an aggregate that specifies a sort operator" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(float, float)': { + 'language': 'sql', 'returns': 'float', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(float)': { + 'sfunc': 'f1', 'stype': 'float', 'sortop': 'pg_catalog.>'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE AGGREGATE sd.a1(float) " \ + "(SFUNC = sd.f1, STYPE = float, SORTOP = OPERATOR(pg_catalog.>))" + + def test_create_aggregate_init_final(self): + "Create an aggregate with an INITCOND and a FINALFUNC" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'function f2(integer)': { + 'language': 'sql', 'returns': 'double precision', + 'source': "SELECT $1::float", 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(integer)': { + 'sfunc': 'f1', 'stype': 'integer', 'initcond': '-1', + 'finalfunc': 'f2'}}) + sql = self.to_sql(inmap) + funcs = sorted(sql[1:3]) + assert fix_indent(funcs[0]) == CREATE_STMT2 + assert fix_indent(funcs[1]) == "CREATE FUNCTION sd.f2(integer) " \ + "RETURNS double precision LANGUAGE sql IMMUTABLE " \ + "AS $_$SELECT $1::float$_$" + assert fix_indent(sql[3]) == "CREATE AGGREGATE sd.a1(integer) " \ + "(SFUNC = sd.f1, STYPE = integer, FINALFUNC = sd.f2, " \ + "INITCOND = '-1')" + + def test_drop_aggregate(self): + "Drop an existing aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE agg1 (integer) " + "(SFUNC = f1, STYPE = integer)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP AGGREGATE sd.agg1(integer)" + assert sql[1] == "DROP FUNCTION sd.f1(integer, integer)" + + def test_create_moving_aggregate(self): + "Create a moving-aggregate mode function" + inmap = self.std_map() + inmap['schema sd'].update( + {'function fadd(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE4, + 'volatility': 'immutable'}, + 'function fsub(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE5, + 'volatility': 'immutable'}, + 'aggregate a1(integer)': { + 'sfunc': 'fadd', 'stype': 'integer', 'initcond': '0', + 'msfunc': 'fadd', 'minvfunc': 'fsub', 'mstype': 'integer', + 'minitcond': '0'}}) + sql = self.to_sql(inmap, [CREATE_STMT4, CREATE_STMT5]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer) (" \ + "SFUNC = sd.fadd, STYPE = integer, INITCOND = '0', " \ + "MSFUNC = sd.fadd, MINVFUNC = sd.fsub, MSTYPE = integer, " \ + "MINITCOND = '0')" + + def test_create_hypothetical_set_aggregate(self): + "Create a hypothetical-set aggregate" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}, + 'aggregate a1(integer ORDER BY integer)': { + 'kind': 'hypothetical', 'sfunc': 'f1', 'stype': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer " \ + "ORDER BY integer) (SFUNC = sd.f1, STYPE = integer, HYPOTHETICAL)" + + def test_create_aggregate_parallel_safe(self): + "Create an aggregate with parallel safety" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}, + 'aggregate a1(integer ORDER BY integer)': { + 'sfunc': 'f1', 'stype': 'integer', 'parallel': 'safe'}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer " \ + "ORDER BY integer) (SFUNC = sd.f1, STYPE = integer, " \ + "PARALLEL = SAFE)" diff --git a/tests/dbobject/test_index.py b/tests/dbobject/test_index.py new file mode 100644 index 0000000..54917e1 --- /dev/null +++ b/tests/dbobject/test_index.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +"""Test indexes""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +CREATE_STMT = "CREATE INDEX t1_idx ON t1 (c1)" +COMMENT_STMT = "COMMENT ON INDEX sd.t1_idx IS 'Test index t1_idx'" + + +class IndexToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created indexes""" + + def test_index_1(self): + "Map a single-column index" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_2(self): + "Map a two-column index" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT)", + "CREATE UNIQUE INDEX t1_idx ON t1 (c1, c2)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1', 'c2'], + 'unique': True}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_3(self): + "Map a table with a unique index and a non-unique GIN index" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 tsvector)", + "CREATE UNIQUE INDEX t1_idx_1 ON t1 (c1, c2)", + "CREATE INDEX t1_idx_2 ON t1 USING gin (c3)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'tsvector'}}], + 'indexes': {'t1_idx_1': {'keys': ['c1', 'c2'], + 'unique': True}, + 't1_idx_2': {'keys': ['c3'], + 'access_method': 'gin'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_partial(self): + "Map a table with a partial index" + dbmap = self.to_map([CREATE_TABLE_STMT, + "CREATE INDEX t1_idx ON t1 (c2) WHERE c1 > 42"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2'], + 'predicate': '(c1 > 42)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_function_simple(self): + "Map an index using a function -- issue #98" + stmts = [CREATE_TABLE_STMT, "CREATE INDEX t1_idx ON t1 ((lower(c2)))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {'lower(c2)': {'type': 'expression'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_function_complex(self): + "Map indexes using nested functions and complex arguments" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 date)", + "CREATE INDEX t1_idx1 ON t1 (substring(c2 from position(" + "'_begin' in c2)), substring(c2 from position('_end' in " + "c2)))", + "CREATE INDEX t1_idx2 ON t1 (extract(month from c3), " + "extract(day from c3))"] + dbmap = self.to_map(stmts) + expr_type = {'type': 'expression'} + key1 = [{"SUBSTRING(c2 FROM POSITION(('_begin'::text) IN (c2)))": + expr_type}, + {"SUBSTRING(c2 FROM POSITION(('_end'::text) IN (c2)))": + expr_type}] + key2 = [{"EXTRACT(month FROM c3)": expr_type}, + {"EXTRACT(day FROM c3)": expr_type}] + if self.db.version < 150000: + key1 = [{'"substring"(c2, "position"(c2, \'_begin\'' + '::text))': expr_type}, + {'"substring"(c2, "position"(c2, \'_end\'' + '::text))': expr_type}] + key2 = [{"date_part('month'::text, c3)": { + 'type': 'expression'}}, + {"date_part('day'::text, c3)": { + 'type': 'expression'}}] + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'indexes': {'t1_idx1': {'keys': key1}, + 't1_idx2': {'keys': key2}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_col_opts(self): + "Map an index with various column options" + stmts = ["CREATE TABLE t1 (c1 cidr, c2 text)", + "CREATE INDEX t1_idx ON t1 (c1 cidr_ops NULLS FIRST, " + "c2 DESC)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'cidr'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {'c1': {'opclass': 'cidr_ops', 'nulls': 'first'}}, + {'c2': {'order': 'desc'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_mixed(self): + "Map indexes using functions, a regular column and expressions" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 text)", + "CREATE INDEX t1_idx ON t1 (btrim(c3, 'x') NULLS FIRST, c1, " + "lower(c2) DESC)", + "CREATE INDEX t1_idx2 ON t1 ((c2 || ', ' || c3), " + "(c3 || ' ' || c2))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': { + 'keys': [{"btrim(c3, 'x'::text)": { + 'type': 'expression', 'nulls': 'first'}}, + 'c1', {'lower(c2)': { + 'type': 'expression', 'order': 'desc'}}]}, + 't1_idx2': { + 'keys': [{"(((c2 || ', '::text) || c3))": { + 'type': 'expression'}}, + {"(((c3 || ' '::text) || c2))": { + 'type': 'expression'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_cluster(self): + "Map a table with an index and cluster on it" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT, + "CLUSTER t1 USING t1_idx"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_comment(self): + "Map an index comment" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['table t1']['indexes']['t1_idx'][ + 'description'] == 'Test index t1_idx' + + def test_map_multicol_index_with_exprs(self): + "Map a multicol index with expressions" + dbmap = self.to_map(["CREATE TABLE holiday (id serial PRIMARY KEY," + "date date NOT NULL, recurring boolean NOT NULL)", + "CREATE UNIQUE INDEX unique_date ON holiday ((" + "CASE WHEN recurring THEN (0)::double precision " + "ELSE date_part('year'::text, date) END), " + "date_part('month'::text, date), " + "date_part('day'::text, date))"]) + fmt = "(\nCASE\n %s\n %s\nEND)" + assert dbmap['schema sd']['table holiday']['indexes'][ + 'unique_date'] == { + 'keys': [ + {fmt % ("WHEN recurring THEN (0)::double precision", + "ELSE date_part('year'::text, date)"): + {'type': 'expression'}}, + {"date_part('month'::text, date)": {'type': 'expression'}}, + {"date_part('day'::text, date)": {'type': 'expression'}}], + 'unique': True} + + +class IndexToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input indexes""" + + def test_create_table_with_index(self): + "Create new table with a single column index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 (c1)" + + def test_add_index(self): + "Add a two-column unique index to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1'], 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON sd.t1 (c2, c1)"] + + def test_add_index_schema(self): + "Add an index to an existing table in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1'], 'unique': True}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON s1.t1 (c2, c1)"] + + def test_add_index_back_compat(self): + "Add a index to an existing table accepting back-compatible spec" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'columns': ['c2', 'c1'], 'unique': True, + 'access_method': 'btree'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON sd.t1 (c2, c1)"] + + def test_bad_index(self): + "Fail on creating an index without columns or expression" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'access_method': 'btree'}}}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_create_partial(self): + "Create a partial index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2'], + 'predicate': '(c1 > 42)'}}}}) + sql = self.to_sql(inmap, [CREATE_TABLE_STMT]) + assert fix_indent(sql[0]) == \ + "CREATE INDEX t1_idx ON sd.t1 (c2) WHERE (c1 > 42)" + + def test_create_index_function(self): + "Create an index which uses a function" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{ + 'lower(c2)': {'type': 'expression'}}]}}}}) + sql = self.to_sql(inmap) + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 (lower(c2))" + + def test_create_index_col_opts(self): + "Create table and an index with column options" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'cidr'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{'c1': { + 'opclass': 'cidr_ops', 'nulls': 'first'}}, 'c2']}}}}) + sql = self.to_sql(inmap) + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 " \ + "(c1 cidr_ops NULLS FIRST, c2)" + + def test_create_index_mixed(self): + "Create indexes using functions, a regular column and expressions" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {"btrim(c3, 'x'::text)": {'type': 'expression', + 'nulls': 'first'}}, 'c1', + {'lower(c2)': {'type': 'expression', 'order': 'desc'}}]}, + 't1_idx2': {'keys': [ + {"(((c2 || ', '::text) || c3))": {'type': 'expression'}}, + {"(((c3 || ' '::text) || c2))": { + 'type': 'expression'}}]}}}}) + sql = sorted(self.to_sql(inmap, stmts)) + assert sql[0] == "CREATE INDEX t1_idx ON sd.t1 (" \ + "btrim(c3, 'x'::text) NULLS FIRST, c1, lower(c2) DESC)" + assert sql[1] == "CREATE INDEX t1_idx2 ON sd.t1 (" \ + "(((c2 || ', '::text) || c3)), (((c3 || ' '::text) || c2)))" + + def test_create_table_with_index_clustered(self): + "Create new table clustered on a single column index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_idx" + + def test_cluster_table_with_index(self): + "Change a table with an index to cluster on it" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "CLUSTER sd.t1 USING t1_idx" + + def test_uncluster_table_with_index(self): + "Change a table clustered on an index to remove cluster" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, "CLUSTER t1 USING t1_idx"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET WITHOUT CLUSTER" + + def test_comment_on_index(self): + "Create a comment for an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], + 'description': 'Test index t1_idx'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_index_comment(self): + "Drop the comment on an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON INDEX sd.t1_idx IS NULL"] + + def test_change_index_comment(self): + "Change existing comment on an index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], + 'description': 'Changed index t1_idx'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON INDEX sd.t1_idx IS 'Changed index t1_idx'"] + + def test_change_index_keys(self): + "Change keys of an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['btrim(c1)']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP INDEX sd.t1_idx", + "CREATE INDEX t1_idx ON sd.t1 (btrim(c1))"] + + def test_change_order_index_keys(self): + "Change keys order of an existing index" + stmts = [CREATE_TABLE_STMT, "CREATE INDEX t1_idx ON t1 (c1, c2)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP INDEX sd.t1_idx", + "CREATE INDEX t1_idx ON sd.t1 (c2, c1)"] diff --git a/tests/dbobject/test_matview.py b/tests/dbobject/test_matview.py new file mode 100644 index 0000000..469f300 --- /dev/null +++ b/tests/dbobject/test_matview.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +"""Test materialized views""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE = "CREATE TABLE t1 (c1 INTEGER, c2 TEXT, c3 INTEGER)" +VIEW_STMT = "SELECT c1, c3 * 2 AS mc3 FROM t1" +CREATE_STMT = "CREATE MATERIALIZED VIEW sd.mv1 AS " + VIEW_STMT +COMMENT_STMT = "COMMENT ON MATERIALIZED VIEW sd.mv1 IS 'Test matview mv1'" +VIEW_DEFN = " SELECT t1.c1,\n t1.c3 * 2 AS mc3\n FROM sd.t1;" + + +class MatViewToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created materialized views""" + + def test_map_view_simple(self): + "Map a created materialized view" + stmts = [CREATE_TABLE, CREATE_STMT] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'with_data': True, + 'depends_on': ['table t1']} + assert dbmap['schema sd']['materialized view mv1'] == expmap + + def test_map_view_comment(self): + "Map a materialized view with a comment" + dbmap = self.to_map([CREATE_TABLE, CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['materialized view mv1'][ + 'description'] == 'Test matview mv1' + + def test_map_view_index(self): + "Map a materialized view with an index" + stmts = [CREATE_TABLE, CREATE_STMT, + "CREATE INDEX idx1 ON mv1 (mc3)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'with_data': True, + 'indexes': {'idx1': {'keys': ['mc3']}}, + 'depends_on': ['table t1']} + assert dbmap['schema sd']['materialized view mv1'] == expmap + + +class MatViewToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input materialized views""" + + def test_create_view(self): + "Create a materialized view" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': "SELECT c1, c3 * 2 AS mc3 FROM sd.t1", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, " \ + "c2 text, c3 integer)" + assert fix_indent(sql[1]) == "CREATE MATERIALIZED VIEW sd.mv1 AS " \ + "SELECT c1, c3 * 2 AS mc3 FROM sd.t1" + + def test_bad_view_map(self): + "Error creating a materialized view with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'mv1': {'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_view(self): + "Drop an existing materialized view with table dependencies" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT)", + "CREATE TABLE t2 (c1 INTEGER, c3 TEXT)", + "CREATE MATERIALIZED VIEW mv1 AS SELECT t1.c1, c2, c3 " + "FROM t1 JOIN t2 ON (t1.c1 = t2.c1)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP MATERIALIZED VIEW sd.mv1" + # can't control which table will be dropped first + drt1 = 1 + drt2 = 2 + if 't1' in sql[2]: + drt1 = 2 + drt2 = 1 + assert sql[drt1] == "DROP TABLE sd.t1" + assert sql[drt2] == "DROP TABLE sd.t2" + + def test_view_with_comment(self): + "Create a materialized view with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_STMT, 'description': "Test matview mv1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_view_index(self): + "Create an index on a materialized view" + stmts = [CREATE_TABLE, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'indexes': {'idx1': {'keys': ['mc3']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE INDEX idx1 ON sd.mv1 (mc3)"] diff --git a/tests/dbobject/test_operator.py b/tests/dbobject/test_operator.py new file mode 100644 index 0000000..3767c47 --- /dev/null +++ b/tests/dbobject/test_operator.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +"""Test operators""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE OPERATOR sd.+ (PROCEDURE = textcat, LEFTARG = text, " \ + "RIGHTARG = text)" +COMMENT_STMT = "COMMENT ON OPERATOR sd.+(text, text) IS 'Test operator +'" + + +class OperatorToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operators""" + + def test_map_operator1(self): + "Map a simple operator" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'procedure': 'textcat'} + assert dbmap['schema sd']['operator +(text, text)'] == expmap + + def test_map_operator_rightarg(self): + "Map a unitary operator with a right argument" + stmts = ["CREATE OPERATOR + (PROCEDURE = upper, RIGHTARG = text)"] + dbmap = self.to_map(stmts) + expmap = {'procedure': 'pg_catalog.upper'} + assert dbmap['schema sd']['operator +(NONE, text)'] == expmap + + def test_map_operator_commutator(self): + "Map an operator with a commutator" + stmts = ["CREATE OPERATOR && (PROCEDURE = int4pl, LEFTARG = integer, " + "RIGHTARG = integer, COMMUTATOR = OPERATOR(sd.&&))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['operator &&(integer, integer)'] == \ + {'procedure': 'int4pl', 'commutator': 'sd.&&'} + + def test_map_operator_hash(self): + "Map an operator with HASHES clause" + stmts = ["CREATE OPERATOR + (PROCEDURE = texteq, LEFTARG = text, " + "RIGHTARG = text, HASHES)"] + dbmap = self.to_map(stmts) + expmap = {'procedure': 'texteq', 'hashes': True} + assert dbmap['schema sd']['operator +(text, text)'] == expmap + + def test_map_operator_comment(self): + "Map a operator comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator +(text, text)'][ + 'description'] == 'Test operator +' + + +class OperatorToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + def test_create_operator1(self): + "Create a simple operator" + inmap = self.std_map() + inmap['schema sd'].update({'operator +(text, text)': { + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_operator_rightarg(self): + "Create a unitary operator with a right argument" + inmap = self.std_map() + inmap['schema sd'].update({'operator +(NONE, text)': { + 'procedure': 'upper'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE OPERATOR sd.+ (" \ + "PROCEDURE = upper, RIGHTARG = text)" + + def test_create_operator_commutator(self): + "Create an operator with a commutator" + inmap = self.std_map() + inmap['schema sd'].update({'operator &&(integer, integer)': { + 'procedure': 'int4pl', 'commutator': 'sd.&&'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE OPERATOR sd.&& (" \ + "PROCEDURE = int4pl, LEFTARG = integer, RIGHTARG = integer, " \ + "COMMUTATOR = OPERATOR(sd.&&))" + + def test_create_operator_in_schema(self): + "Create a operator within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator +(text, text)': { + 'procedure': 'textcat'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE OPERATOR s1.+ " \ + "(PROCEDURE = textcat, LEFTARG = text, RIGHTARG = text)" + + def test_drop_operator(self): + "Drop an existing operator" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP OPERATOR sd.+(text, text)"] + + def test_operator_with_comment(self): + "Create a operator with a comment" + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Test operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_operator(self): + "Create a comment for an existing operator" + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Test operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_operator_comment(self): + "Drop a comment on an existing operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'returns': 'integer', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON OPERATOR sd.+(text, text) IS NULL"] + + def test_change_operator_comment(self): + "Change existing comment on a operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Changed operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON OPERATOR sd.+(text, text) IS " + "'Changed operator +'"] diff --git a/tests/dbobject/test_operclass.py b/tests/dbobject/test_operclass.py new file mode 100644 index 0000000..bda1330 --- /dev/null +++ b/tests/dbobject/test_operclass.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +"""Test operator classes""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TYPE_STMT = """ +CREATE TYPE myint; +CREATE FUNCTION myintin(cstring) RETURNS myint AS 'int4in' LANGUAGE internal; +CREATE FUNCTION myintout(myint) RETURNS cstring AS 'int4out' LANGUAGE internal; +CREATE TYPE myint (INPUT = myintin, OUTPUT = myintout); +CREATE FUNCTION myinteq(myint,myint) RETURNS boolean AS 'int4eq' + LANGUAGE internal; +CREATE FUNCTION myintlt(myint,myint) RETURNS boolean AS 'int4lt' + LANGUAGE internal; +CREATE OPERATOR < (PROCEDURE = myintlt, LEFTARG = myint, RIGHTARG = myint); +CREATE OPERATOR = (PROCEDURE = myinteq, LEFTARG = myint, RIGHTARG = myint); +CREATE FUNCTION btmyintcmp(myint,myint) RETURNS integer AS 'btint4cmp' + LANGUAGE internal; +""" + +CREATE_STMT = "CREATE OPERATOR CLASS oc1 FOR TYPE integer USING btree " \ + "AS OPERATOR 1 <, OPERATOR 3 =, FUNCTION 1 btint4cmp(integer,integer)" +CREATE_STMT_LONG = "CREATE OPERATOR CLASS sd.oc1 FOR TYPE integer " \ + "USING btree AS OPERATOR 1 <(integer,integer), " \ + "OPERATOR 3 =(integer,integer), FUNCTION 1 btint4cmp(integer,integer)" +COMMENT_STMT = "COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS " \ + "'Test operator class oc1'" + + +class OperatorClassToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operator classes""" + + superuser = True + + def test_map_operclass1(self): + "Map an operator class" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['operator class oc1 using btree'] == \ + {'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}} + + def test_map_operclass_default(self): + "Map a default operator class" + stmts = [CREATE_TYPE_STMT, + "CREATE OPERATOR CLASS oc1 DEFAULT FOR TYPE sd.myint " + "USING btree AS OPERATOR 1 <, OPERATOR 3 =, " + "FUNCTION 1 btmyintcmp(sd.myint,sd.myint)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['operator class oc1 using btree'] == \ + {'type': 'myint', 'default': True, + 'operators': {1: 'sd.<(sd.myint,sd.myint)', + 3: 'sd.=(sd.myint,sd.myint)'}, + 'functions': {1: 'sd.btmyintcmp(sd.myint,sd.myint)'}} + + def test_map_operclass_comment(self): + "Map an operator class comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator class oc1 using btree'][ + 'description'] == 'Test operator class oc1' + + +class OperatorClassToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + @pytest.mark.xfail + def test_create_operclass1(self): + "Create an operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_LONG + + def test_create_operclass_default(self): + "Create a default operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'default': True, 'type': 'myint', + 'operators': {1: 'sd.<(sd.myint,sd.myint)', + 3: 'sd.=(sd.myint,sd.myint)'}, + 'functions': {1: 'sd.btmyintcmp(sd.myint,sd.myint)'}}}) + sql = self.to_sql(inmap, [CREATE_TYPE_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE OPERATOR CLASS sd.oc1 DEFAULT " \ + "FOR TYPE sd.myint USING btree AS OPERATOR 1 " \ + "sd.<(sd.myint,sd.myint), OPERATOR 3 sd.=(sd.myint,sd.myint), "\ + "FUNCTION 1 sd.btmyintcmp(sd.myint,sd.myint)" + + @pytest.mark.xfail + def test_create_operclass_in_schema(self): + "Create a operator within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE OPERATOR CLASS s1.oc1 FOR " \ + "TYPE integer USING btree AS OPERATOR 1 <(integer,integer), " \ + "OPERATOR 3 =(integer,integer), " \ + "FUNCTION 1 btint4cmp(integer,integer)" + + def test_drop_operclass(self): + "Drop an existing operator" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP OPERATOR CLASS sd.oc1 USING btree", + "DROP OPERATOR FAMILY sd.oc1 USING btree"] + + @pytest.mark.xfail + def test_operclass_with_comment(self): + "Create an operator class with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Test operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_LONG + assert sql[1] == COMMENT_STMT + + def test_comment_on_operclass(self): + "Create a comment for an existing operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Test operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_operclass_comment(self): + "Drop the existing comment on an operator class" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS NULL"] + + def test_change_operclass_comment(self): + "Change existing comment on an operator class" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Changed operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS " + "'Changed operator class oc1'"] diff --git a/tests/dbobject/test_operfamily.py b/tests/dbobject/test_operfamily.py new file mode 100644 index 0000000..ea1fcb3 --- /dev/null +++ b/tests/dbobject/test_operfamily.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""Test operator families""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE OPERATOR FAMILY sd.of1 USING btree" +COMMENT_STMT = "COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS " \ + "'Test operator family of1'" + + +class OperatorFamilyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operators""" + + superuser = True + + def test_map_operfam(self): + "Map an operator family" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['operator family of1 using btree'] == {} + + def test_map_operfam_comment(self): + "Map an operator family comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator family of1 using btree'][ + 'description'] == 'Test operator family of1' + + +class OperatorFamilyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + def test_create_operfam(self): + "Create an operator family" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': {}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_operfam_in_schema(self): + "Create an operator family within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator family of1 using btree': {}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == \ + "CREATE OPERATOR FAMILY s1.of1 USING btree" + + def test_drop_operfam(self): + "Drop an existing operator family" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP OPERATOR FAMILY sd.of1 USING btree"] + + def test_operfam_with_comment(self): + "Create an operator family with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Test operator family of1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_operfam(self): + "Create a comment for an existing operator family" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Test operator family of1'}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_operfam_comment(self): + "Drop a comment on an existing operator family" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS NULL"] + + def test_change_operfam_comment(self): + "Change existing comment on a operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Changed operator family of1'}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS " + "'Changed operator family of1'"] diff --git a/tests/dbobject/test_owner.py b/tests/dbobject/test_owner.py new file mode 100644 index 0000000..1dbd777 --- /dev/null +++ b/tests/dbobject/test_owner.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +"""Test object ownership + +The majority of other tests exclude owner information. These +explicitly request it. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +SOURCE1 = "SELECT 'dummy'::text" +SOURCE2 = "SELECT $1 * $2" +CREATE_FUNC = "CREATE FUNCTION sd.f1() RETURNS text LANGUAGE sql IMMUTABLE " \ + "AS $_$%s$_$" % SOURCE1 +CREATE_TYPE = "CREATE TYPE sd.t1 AS (x integer, y integer)" + + +class OwnerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of object owner information""" + + def test_map_type(self): + "Map a composite type" + dbmap = self.to_map([CREATE_TYPE], no_owner=False) + expmap = {'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}], + 'owner': self.db.user} + assert dbmap['schema sd']['type t1'] == expmap + + def test_map_table(self): + "Map a table" + dbmap = self.to_map([CREATE_TABLE], no_owner=False) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'owner': self.db.user} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_function(self): + "Map a function" + dbmap = self.to_map([CREATE_FUNC], no_owner=False) + expmap = {'language': 'sql', 'returns': 'text', 'owner': self.db.user, + 'source': SOURCE1, 'volatility': 'immutable'} + assert dbmap['schema sd']['function f1()'] == expmap + + +class OwnerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of owner object information""" + + def test_create_type(self): + "Create a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}], + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TYPE + assert sql[1] == "ALTER TYPE sd.t1 OWNER TO %s" % self.db.user + + def test_create_table(self): + "Create a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE + assert sql[1] == "ALTER TABLE sd.t1 OWNER TO %s" % self.db.user + + def test_create_function(self): + "Create a function in a schema and a comment" + inmap = self.std_map() + inmap.update({'schema s1': {'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'description': 'Test function'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + # skip first two statements as those are tested elsewhere + assert sql[2] == "ALTER FUNCTION s1.f1() OWNER TO %s" % self.db.user + # to verify correct order of invocation of ownable and commentable + assert sql[3] == "COMMENT ON FUNCTION s1.f1() IS 'Test function'" + + def test_create_function_default_args(self): + "Create a function with default arguments" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, INOUT integer)': { + 'allargs': 'integer, INOUT integer DEFAULT 1', + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == \ + "CREATE FUNCTION sd.f1(integer, INOUT integer DEFAULT 1) " \ + "RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + assert sql[2] == "ALTER FUNCTION sd.f1(integer, INOUT integer) " \ + "OWNER TO %s" % self.db.user + + def test_change_table_owner(self): + "Change the owner of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': 'someuser'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert sql[0] == "ALTER TABLE sd.t1 OWNER TO someuser" + + def test_change_table_owner_delim(self): + "Change the owner of a table with delimited identifiers" + inmap = self.std_map() + inmap.update({'schema a-schema': {'table a-table': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': 'someuser'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA \"a-schema\"", + "CREATE TABLE \"a-schema\".\"a-table\" (" + "c1 integer, c2 text)"]) + assert sql[0] == "ALTER TABLE \"a-schema\".\"a-table\" OWNER TO " \ + "someuser" diff --git a/tests/dbobject/test_privs.py b/tests/dbobject/test_privs.py new file mode 100644 index 0000000..1a230c6 --- /dev/null +++ b/tests/dbobject/test_privs.py @@ -0,0 +1,446 @@ +# -*- coding: utf-8 -*- +"""Test object privileges + +The majority of other tests exclude access privileges. These +explicitly request it. In addition, the roles 'user1' and 'user2' +are created if they don't exist. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_TABLE = "CREATE TABLE t1 (c1 integer, c2 text)" +SOURCE1 = "SELECT 'dummy'::text" +CREATE_FUNC = "CREATE FUNCTION f1() RETURNS text LANGUAGE sql IMMUTABLE AS " \ + "$_$%s$_$" % SOURCE1 +CREATE_FDW = "CREATE FOREIGN DATA WRAPPER fdw1" +CREATE_FS = "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1" +GRANT_SELECT = "GRANT SELECT ON TABLE sd.t1 TO %s" +GRANT_INSUPD = "GRANT INSERT, UPDATE ON TABLE sd.t1 TO %s" + + +def check_extra_users(db): + "Check existence of extra test users" + for user in ['user1', 'user2']: + row = db.fetchone("SELECT 1 FROM pg_roles WHERE rolname = %s", (user,)) + if row is None: + db.execute_commit("CREATE ROLE %s" % user) + + +class PrivilegeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of object privilege information""" + + def setUp(self): + super(DatabaseToMapTestCase, self).setUp() + check_extra_users(self.db) + + def test_map_schema(self): + "Map a schema with some GRANTs" + stmts = ["CREATE SCHEMA s1", "GRANT USAGE ON SCHEMA s1 TO PUBLIC", + "GRANT CREATE, USAGE ON SCHEMA s1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['usage']}, + {'user1': ['all']}]}) + assert dbmap['schema s1'] == expmap + + def test_map_table(self): + "Map a table with various GRANTs" + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', GRANT_INSUPD % 'user1', + "GRANT REFERENCES, TRIGGER ON t1 TO user2 WITH GRANT OPTION"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}, + {'user2': [{'trigger': {'grantable': True}}, + {'references': { + 'grantable': True}}]}]}) + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_column(self): + "Map a table with GRANTs on column" + self.maxDiff = None + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', + "GRANT INSERT (c1, c2) ON t1 TO user1", + "GRANT INSERT (c2), UPDATE (c2) ON t1 TO user2"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'columns': [ + {'c1': {'type': 'integer', 'privileges': [{'user1': ['insert']}]}}, + {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}, {'user2': [ + 'insert', 'update']}]}}], 'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['select']}]}) + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_sequence(self): + "Map a sequence with various GRANTs" + stmts = ["CREATE SEQUENCE seq1", + "GRANT SELECT ON SEQUENCE seq1 TO PUBLIC", + "GRANT USAGE, UPDATE ON SEQUENCE seq1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, + 'cache_value': 1, 'privileges': [ + {self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['usage', 'update']}]}) + assert dbmap['schema sd']['sequence seq1'] == expmap + + def test_map_view(self): + "Map a view with various GRANTs" + stmts = ["CREATE VIEW v1 AS SELECT now()::date AS today", + "GRANT SELECT ON v1 TO PUBLIC", + "GRANT REFERENCES ON v1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges( + {'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT now()::date AS today;", + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['references']}]}) + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_function(self): + "Map a function with a GRANT and REVOKE from PUBLIC" + stmts = [CREATE_FUNC, "REVOKE ALL ON FUNCTION f1() FROM PUBLIC", + "GRANT EXECUTE ON FUNCTION f1() TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable', + 'privileges': [{self.db.user: ['execute']}, + {'user1': ['execute']}]} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_fd_wrapper(self): + "Map a foreign data wrapper with a GRANT" + stmts = [CREATE_FDW, + "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = self.sort_privileges({'privileges': + [{self.db.user: ['usage']}, + {'PUBLIC': ['usage']}]}) + assert dbmap['foreign data wrapper fdw1'] == expmap + + def test_map_server(self): + "Map a foreign server with a GRANT" + stmts = [CREATE_FDW, CREATE_FS, + "GRANT USAGE ON FOREIGN SERVER fs1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = {'privileges': [{self.db.user: ['usage']}, + {'user1': ['usage']}]} + assert dbmap['foreign data wrapper fdw1']['server fs1'] == expmap + + def test_map_foreign_table(self): + "Map a foreign table with various GRANTs" + stmts = [CREATE_FDW, CREATE_FS, + "CREATE FOREIGN TABLE ft1 (c1 integer, c2 text) SERVER fs1", + "GRANT SELECT ON ft1 TO PUBLIC", + "GRANT INSERT, UPDATE ON ft1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]} + assert dbmap['schema sd']['foreign table ft1'] == \ + self.sort_privileges(expmap) + + +class PrivilegeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of privilege information (GRANTs)""" + + def setUp(self): + super(InputMapToSqlTestCase, self).setUp() + check_extra_users(self.db) + + def test_create_schema(self): + "Create a schema with various privileges" + inmap = self.std_map() + inmap.update({'schema s1': { + 'owner': self.db.user, 'privileges': [{ + self.db.user: ['all']}, {'PUBLIC': ['usage', 'create']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE SCHEMA + # sql[1] = ALTER SCHEMA OWNER + assert sql[2] == "GRANT ALL ON SCHEMA s1 TO %s" % self.db.user + assert sql[3] == "GRANT ALL ON SCHEMA s1 TO PUBLIC" + + def test_schema_new_grant(self): + "Grant privileges on an existing schema" + inmap = self.std_map() + inmap.update({'schema s1': { + 'owner': self.db.user, 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['create']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE SCHEMA s1"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON SCHEMA s1 TO %s" % self.db.user + assert sql[1] == "GRANT CREATE ON SCHEMA s1 TO PUBLIC" + + def test_create_table(self): + "Create a table with various privileges" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}, + {'user2': [{'trigger': {'grantable': True}}, + {'references': {'grantable': True}}]}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[3] == GRANT_SELECT % 'PUBLIC' + assert sql[4] == GRANT_INSUPD % 'user1' + assert sql[5] == "GRANT TRIGGER, REFERENCES ON TABLE sd.t1 " \ + "TO user2 WITH GRANT OPTION" + + def test_create_column_grants(self): + "Create a table with colum-level privileges" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'privileges': [{'user1': [ + 'insert']}]}}, {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}, {'user2': ['insert', 'update']}]}}], + 'owner': self.db.user, 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}]}}) + sql = self.to_sql(inmap) + assert len(sql) == 7 + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[3] == GRANT_SELECT % 'PUBLIC' + assert sql[4] == "GRANT INSERT (c1) ON TABLE sd.t1 TO user1" + assert sql[5] == "GRANT INSERT (c2) ON TABLE sd.t1 TO user1" + assert sql[6] == "GRANT INSERT (c2), UPDATE (c2) ON TABLE sd.t1 " \ + "TO user2" + + def test_table_new_grant(self): + "Grant select privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert len(sql) == 2 + sql = sorted(sql) + assert sql[0] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[1] == GRANT_SELECT % 'user1' + + def test_table_change_grant(self): + "Grant select privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = self.to_sql(inmap, [CREATE_TABLE, GRANT_SELECT % 'user1']) + assert len(sql) == 3 + assert sorted(sql) == [GRANT_INSUPD % 'user1', GRANT_SELECT % 'PUBLIC', + "REVOKE SELECT ON TABLE sd.t1 FROM user1"] + + def test_column_change_grants(self): + "Change existing colum-level privileges" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c1': { + 'type': 'integer', 'privileges': [ + {'user1': ['insert']}, {'user2': ['insert', 'update']}]}}, + {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}]}}], + 'owner': self.db.user, 'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', + "GRANT INSERT (c1, c2) ON t1 TO user1", + "GRANT INSERT (c2), UPDATE (c2) ON t1 TO user2"] + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert sql[0] == "GRANT INSERT (c1), UPDATE (c1) ON TABLE sd.t1 " \ + "TO user2" + assert sql[1] == "REVOKE INSERT (c2), UPDATE (c2) ON TABLE sd.t1 " \ + "FROM user2" + + def test_table_revoke_all(self): + "Revoke all privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user}}) + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', GRANT_INSUPD % 'user1'] + sql = sorted(self.to_sql(inmap, stmts)) + assert len(sql) == 3 + assert sql[0] == "REVOKE ALL ON TABLE sd.t1 FROM %s" % self.db.user + assert sql[1] == "REVOKE INSERT, UPDATE ON TABLE sd.t1 FROM user1" + assert sql[2] == "REVOKE SELECT ON TABLE sd.t1 FROM PUBLIC" + + def test_create_sequence(self): + "Create a sequence with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE SEQUENCE + # sql[1] = ALTER SEQUENCE OWNER + assert sql[2] == "GRANT ALL ON SEQUENCE sd.seq1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON SEQUENCE sd.seq1 TO PUBLIC" + + def test_sequence_new_grant(self): + "Grant privileges on an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE SEQUENCE seq1"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON SEQUENCE sd.seq1 TO %s" % self.db.user + assert sql[1] == "GRANT SELECT ON SEQUENCE sd.seq1 TO PUBLIC" + + def test_create_view(self): + "Create a view with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'definition': " SELECT now()::date AS today;", + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE VIEW + # sql[1] = ALTER VIEW OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.v1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON TABLE sd.v1 TO user1" + + def test_view_new_grant(self): + "Grant privileges on an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT now()::date AS today;", + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE VIEW v1 AS " + "SELECT now()::date AS today"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON TABLE sd.v1 TO %s" % self.db.user + assert sql[1] == "GRANT SELECT ON TABLE sd.v1 TO user1" + + def test_create_function(self): + "Create a function with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['execute']}]}}) + sql = self.to_sql(inmap) + # sql[0:1] = SET, CREATE FUNCTION + # sql[2] = ALTER FUNCTION OWNER + assert sql[3] == "GRANT EXECUTE ON FUNCTION sd.f1() TO %s" % ( + self.db.user) + assert sql[4] == "GRANT EXECUTE ON FUNCTION sd.f1() TO PUBLIC" + + def test_function_new_grant(self): + "Grant privileges on an existing function" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['execute']}]}}) + sql = self.to_sql(inmap, [CREATE_FUNC]) + assert len(sql) == 2 + sql = sorted(sql) + # assumes self.db.user > PUBLIC + assert sql[0] == "GRANT EXECUTE ON FUNCTION sd.f1() TO PUBLIC" + assert sql[1] == "GRANT EXECUTE ON FUNCTION sd.f1() TO %s" % ( + self.db.user) + + def test_create_fd_wrapper(self): + "Create a foreign data wrapper with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['usage']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE FDW + # sql[1] = ALTER FDW OWNER + assert sql[2] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 " \ + "TO %s" % self.db.user + assert sql[3] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC" + + def test_fd_wrapper_new_grant(self): + "Grant privileges on an existing foreign data wrapper" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['usage']}]}}) + sql = sorted(self.to_sql(inmap, [CREATE_FDW], superuser=True)) + assert len(sql) == 2 + # assumes self.db.user > PUBLIC + assert sql[0] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC" + assert sql[1] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 " \ + "TO %s" % self.db.user + + def test_create_server(self): + "Create a foreign server with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user2': ['usage']}]}}}) + sql = self.to_sql(inmap, [CREATE_FDW], superuser=True) + # sql[0] = CREATE SERVER + # sql[1] = ALTER SERVER OWNER + assert sql[2] == "GRANT USAGE ON FOREIGN SERVER fs1 TO %s" % \ + self.db.user + assert sql[3] == "GRANT USAGE ON FOREIGN SERVER fs1 TO user2" + + def test_server_new_grant(self): + "Grant privileges on an existing foreign server" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user2': ['usage']}]}}}) + sql = sorted(self.to_sql(inmap, [CREATE_FDW, CREATE_FS], + superuser=True)) + assert len(sql) == 2 + assert sql[0] == "GRANT USAGE ON FOREIGN SERVER fs1 TO %s" % \ + self.db.user + assert sql[1] == "GRANT USAGE ON FOREIGN SERVER fs1 TO user2" + + def test_create_foreign_table(self): + "Create a foreign table with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = self.to_sql(inmap, [CREATE_FDW, CREATE_FS], superuser=True) + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.ft1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON TABLE sd.ft1 TO PUBLIC" + assert sql[4] == "GRANT INSERT, UPDATE ON TABLE sd.ft1 TO user1" + + def test_foreign_table_new_grant(self): + "Grant privileges on an existing foreign table" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = sorted(self.to_sql(inmap, [ + CREATE_FDW, CREATE_FS, + "CREATE FOREIGN TABLE ft1 (c1 integer, c2 text) SERVER fs1"], + superuser=True)) + assert len(sql) == 3 + assert sql[0] == "GRANT ALL ON TABLE sd.ft1 TO %s" % self.db.user + assert sql[1] == "GRANT INSERT, UPDATE ON TABLE sd.ft1 TO user1" + assert sql[2] == "GRANT SELECT ON TABLE sd.ft1 TO PUBLIC" diff --git a/tests/dbobject/test_rule.py b/tests/dbobject/test_rule.py new file mode 100644 index 0000000..90e0304 --- /dev/null +++ b/tests/dbobject/test_rule.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +"""Test rules""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +CREATE_STMT = "CREATE RULE r1 AS ON %s TO sd.t1 DO %s" +COMMENT_STMT = "COMMENT ON RULE r1 ON sd.t1 IS 'Test rule r1'" + + +class RuleToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing rules""" + + def test_map_rule_nothing(self): + "Map a rule to do nothing" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules'] == \ + {'r1': {'event': 'insert', 'actions': 'NOTHING'}} + + def test_map_rule_instead(self): + "Map rule with an INSTEAD action" + stmts = [CREATE_TABLE_STMT, + CREATE_STMT % ('UPDATE', 'INSTEAD NOTHING')] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules'] == \ + {'r1': {'event': 'update', 'instead': True, 'actions': 'NOTHING'}} + + def test_map_rule_conditional(self): + "Map rule with a qualification" + stmts = [CREATE_TABLE_STMT, + "CREATE RULE r1 AS ON DELETE TO t1 WHERE OLD.c1 < 1000 " + "DO INSERT INTO t1 VALUES (OLD.c1 + 1000, OLD.c2)"] + dbmap = self.to_map(stmts) + fmt = " %s\n %s" + action = fmt % ("INSERT INTO sd.t1 (c1, c2)", + " VALUES ((old.c1 + 1000), old.c2)") + expmap = {'r1': {'event': 'delete', 'condition': "(old.c1 < 1000)", + 'actions': action}} + assert dbmap['schema sd']['table t1']['rules'] == expmap + + def test_map_rule_multi_actions(self): + "Map rule with multiple actions" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ( + 'UPDATE', "(INSERT INTO t1 (c1) VALUES (old.c1 + 100); " + "INSERT INTO t1 (c1) VALUES (old.c1 + 200))")] + dbmap = self.to_map(stmts) + ins = "INSERT INTO sd.t1 (c1)" + fmt = "( %s\n %s\n %s\n %s\n)" + actions = fmt % (ins, " VALUES ((old.c1 + 100));", + ins, " VALUES ((old.c1 + 200));") + expmap = {'r1': {'event': 'update', 'actions': actions}} + assert dbmap['schema sd']['table t1']['rules'] == expmap + + def test_map_rule_comment(self): + "Map a rule comment" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules']['r1'][ + 'description'] == 'Test rule r1' + + +class RuleToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input rules""" + + def test_create_rule_nothing(self): + "Create a rule" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == CREATE_STMT % ('INSERT', 'NOTHING') + + def test_create_rule_instead(self): + "Create a rule with an INSTEAD action" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'update', 'instead': True, + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON UPDATE TO sd.t1 " \ + "DO INSTEAD NOTHING" + + def test_create_rule_conditional(self): + "Create a rule with qualification" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'delete', + 'condition': "old.c1 < 1000", + 'actions': "INSERT INTO t1 VALUES (" + "old.c1 + 1000, old.c2)"}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON DELETE TO sd.t1 " \ + "WHERE old.c1 < 1000 DO INSERT INTO t1 VALUES " \ + "(old.c1 + 1000, old.c2)" + + def test_create_rule_multi_actions(self): + "Create a rule with multiple actions" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'update', 'actions': + "(INSERT INTO t1 VALUES (old.c1 + 100); " + "INSERT INTO t1 VALUES (old.c1 + 200));)"}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON UPDATE " \ + "TO sd.t1 DO (INSERT INTO t1 VALUES (old.c1 + 100); " \ + "INSERT INTO t1 VALUES (old.c1 + 200));)" + + def test_create_rule_in_schema(self): + "Create a rule within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': { + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', + 'actions': 'NOTHING'}}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON INSERT TO s1.t1 " \ + "DO NOTHING" + + def test_drop_rule(self): + "Drop an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP RULE r1 ON sd.t1"] + + def test_drop_rule_table(self): + "Drop an existing rule and the related table" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP RULE r1 ON sd.t1" + assert sql[1] == "DROP TABLE sd.t1" + + def test_rule_with_comment(self): + "Create a rule with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'description': 'Test rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert sql[2] == COMMENT_STMT + + def test_comment_on_rule(self): + "Create a comment on an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'description': 'Test rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_rule_comment(self): + "Drop the comment on an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON RULE r1 ON sd.t1 IS NULL"] + + def test_change_rule_comment(self): + "Change existing comment on a rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', + 'description': 'Changed rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON RULE r1 ON sd.t1 IS 'Changed rule r1'"] diff --git a/tests/dbobject/test_schema.py b/tests/dbobject/test_schema.py new file mode 100644 index 0000000..cb6a9a1 --- /dev/null +++ b/tests/dbobject/test_schema.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +"""Test schemas""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_STMT = "CREATE SCHEMA s1" +COMMENT_STMT = "COMMENT ON SCHEMA s1 IS 'Test schema s1'" + + +class SchemaToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created schemas""" + + def test_map_schema(self): + "Map a created schema" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema s1'] == {} + + def test_map_schema_comment(self): + "Map a schema comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema s1'] == {'description': 'Test schema s1'} + + def test_map_select_schema(self): + "Map a single schema when three schemas exist" + stmts = [CREATE_STMT, "CREATE SCHEMA s2", "CREATE SCHEMA s3"] + dbmap = self.to_map(stmts, schemas=['s2']) + assert 'schema s1' not in dbmap + assert dbmap['schema s2'] == {} + assert 'schema s3' not in dbmap + + +class SchemaToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input schemas""" + + def base_schmap(self): + return {'schema s1': {'description': 'Test schema s1'}} + + def test_create_schema(self): + "Create a new schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT] + + def test_bad_schema_map(self): + "Error creating a schema with a bad map" + with pytest.raises(KeyError): + self.to_sql({'s1': {}}) + + def test_drop_schema(self): + "Drop an existing schema" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP SCHEMA s1"] + + def test_rename_schema(self): + "Rename an existing schema" + inmap = self.std_map() + inmap.update({'schema s2': {'oldname': 's1'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER SCHEMA s1 RENAME TO s2"] + + def test_bad_rename_schema(self): + "Error renaming a non-existing schema" + inmap = self.std_map() + inmap.update({'schema s2': {'oldname': 's3'}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_schema_with_comment(self): + "Create a schema with a comment" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT, COMMENT_STMT] + + def test_comment_on_schema(self): + "Create a comment for an existing schema" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_schema_comment(self): + "Drop a comment on an existing schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SCHEMA s1 IS NULL"] + + def test_change_schema_comment(self): + "Change existing comment on a schema" + inmap = self.std_map() + inmap.update({'schema s1': {'description': 'Changed schema s1'}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SCHEMA s1 IS 'Changed schema s1'"] + + +class SchemaUndoSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation to revert schemas""" + + def base_schmap(self): + return {'schema s1': {'description': 'Test schema s1'}} + + def test_undo_create_schema(self): + "Revert a schema creation" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + sql = self.to_sql(inmap, revert=True) + assert sql == ["DROP SCHEMA s1"] + + def test_undo_drop_schema(self): + "Revert dropping a schema" + sql = self.to_sql(self.std_map(), [CREATE_STMT], revert=True) + assert sql[0] == CREATE_STMT + + def test_undo_comment_on_schema(self): + "Revert creating comment on a schema" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap, [CREATE_STMT], revert=True) + assert sql == ["COMMENT ON SCHEMA s1 IS NULL"] + + def test_undo_drop_schema_comment(self): + "Revert dropping comment on a schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts, revert=True) + assert sql == [COMMENT_STMT] diff --git a/tests/dbobject/test_sequence.py b/tests/dbobject/test_sequence.py new file mode 100644 index 0000000..4353511 --- /dev/null +++ b/tests/dbobject/test_sequence.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +"""Test sequences""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE SEQUENCE seq1" +CREATE_STMT_FULL = "CREATE SEQUENCE sd.seq1 %sSTART WITH 1 INCREMENT BY 1 " \ + "NO MINVALUE NO MAXVALUE CACHE 1" +COMMENT_STMT = "COMMENT ON SEQUENCE sd.seq1 IS 'Test sequence seq1'" + + +class SequenceToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created sequences""" + + def test_map_sequence_simple(self): + "Map a created sequence" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1} + assert dbmap['schema sd']['sequence seq1'] == expmap + + def test_map_sequence_comment(self): + "Map a sequence with a comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['sequence seq1']['description'] == \ + 'Test sequence seq1' + + +class SequenceToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input sequences""" + + def test_create_sequence_simple(self): + "Create a sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'data_type': 'integer'}}) + sql = self.to_sql(inmap) + mod = 'AS integer ' + assert fix_indent(sql[0]) == CREATE_STMT_FULL % mod + + def test_create_sequence_in_schema(self): + "Create a sequence within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE SEQUENCE s1.seq1 START WITH 1 " \ + "INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" + + def test_bad_sequence_map(self): + "Error creating a sequence with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_sequence(self): + "Drop an existing sequence" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP SEQUENCE sd.seq1"] + + def test_no_drop_owned_sequence(self): + "Don't drop a sequence owned by a table column" + sql = self.to_sql(self.std_map(), + ["CREATE TABLE t1 (c1 SERIAL, c2 text)"]) + assert sql == ["DROP TABLE sd.t1"] + + def test_rename_sequence(self): + "Rename an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq2': { + 'oldname': 'seq1', 'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER SEQUENCE sd.seq1 RENAME TO seq2"] + + def test_bad_rename_sequence(self): + "Error renaming a non-existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq2': { + 'oldname': 'seq3', 'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, 'cache_value': 1}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_change_sequence(self): + "Change sequence attributes" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 5, 'increment_by': 10, 'max_value': None, + 'min_value': None, 'cache_value': 30}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "ALTER SEQUENCE sd.seq1 START WITH 5 " \ + "INCREMENT BY 10 CACHE 30" + + def test_sequence_with_comment(self): + "Create a sequence with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Test sequence seq1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_FULL % '' + assert sql[1] == COMMENT_STMT + + def test_comment_on_sequence(self): + "Create a comment for an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Test sequence seq1"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_sequence_comment(self): + "Drop the comment on an existing sequence" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SEQUENCE sd.seq1 IS NULL"] + + def test_change_sequence_comment(self): + "Change existing comment on a sequence" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Changed sequence seq1"}}) + sql = self.to_sql(inmap, stmts) + assert sql, ["COMMENT ON SEQUENCE sd.seq1 IS 'Changed sequence seq1'"] + + +class SequenceUndoSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation to revert sequences""" + + def test_undo_create_sequence(self): + "Revert a sequence creation" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, revert=True) + assert sql == ["DROP SEQUENCE sd.seq1"] + + def test_undo_create_sequence_in_schema(self): + "Revert creating a sequence in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"], revert=True) + assert sql == ["DROP SEQUENCE s1.seq1"] + + def test_undo_drop_sequence(self): + "Revert dropping a sequence" + sql = self.to_sql(self.std_map(), [CREATE_STMT], revert=True) + assert fix_indent(sql[0]) == CREATE_STMT_FULL % '' + + def test_undo_change_sequence(self): + "Revert changing sequence attributes" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 5, 'increment_by': 10, 'max_value': None, + 'min_value': None, 'cache_value': 30}}) + sql = self.to_sql(inmap, [CREATE_STMT], revert=True) + assert fix_indent(sql[0]) == "ALTER SEQUENCE sd.seq1 START WITH 1 " \ + "INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" diff --git a/tests/dbobject/test_static.py b/tests/dbobject/test_static.py new file mode 100644 index 0000000..009fc2d --- /dev/null +++ b/tests/dbobject/test_static.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +"""Test loading of data from and into static tables""" +import os + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +FILE_PATH = 'table.t1.data' +TABLE_DATA = [(1, 'abc'), (2, 'def'), (3, 'ghi')] +TABLE_DATA2 = [(1, 'abc', 'row 1'), (3, 'ghi', 'row 2'), (2, 'def', 'row 3'), + (3, 'def', 'row 4')] + + +class StaticTableToMapTestCase(DatabaseToMapTestCase): + """Test mapping and copying out of created tables""" + + def tearDown(self): + self.remove_tempfiles() + + def test_copy_static_table(self): + "Copy a two-column table to a file" + self.db.execute(CREATE_STMT) + for row in TABLE_DATA: + self.db.execute("INSERT INTO t1 VALUES (%s, %s)", row) + cfg = {'datacopy': {'schema sd': ['t1']}} + dbmap = self.to_map([], config=cfg) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]} + recs = [] + with open(os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH)) as f: + for line in f: + (c1, c2) = line.split(',') + recs.append((int(c1), c2.rstrip())) + assert recs == TABLE_DATA + + def test_copy_static_table_pk(self): + "Copy a table that has a primary key" + self.db.execute("CREATE TABLE t1 (c1 integer, c2 char(3), c3 text," + "PRIMARY KEY (c2, c1))") + for row in TABLE_DATA2: + self.db.execute("INSERT INTO t1 VALUES (%s, %s, %s)", row) + cfg = {'datacopy': {'schema sd': ['t1']}} + dbmap = self.to_map([], config=cfg) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(3)', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}} + recs = [] + with open(os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH)) as f: + for line in f: + (c1, c2, c3) = line.split(',') + recs.append((int(c1), c2, c3.rstrip())) + assert recs == sorted(TABLE_DATA2) + + +class StaticTableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of data import statements""" + + def test_load_static_table(self): + "Truncate and import a two-column table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + cfg = {'datacopy': {'schema sd': ['t1']}} + sql = self.to_sql(inmap, [CREATE_STMT], config=cfg) + copy_stmt = ("\\copy ", 'sd.t1', " from '", + os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH), "' csv") + assert sql[0] == "TRUNCATE ONLY sd.t1" + assert sql[1] == copy_stmt + + def test_load_static_table_fk(self): + "Truncate and import a table which has a foreign key dependency" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, c2 integer REFERENCES t1, " + "c3 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['pc1']}}}, 'table t2': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t2_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}}}) + cfg = {'datacopy': {'schema sd': ['t1']}} + sql = self.to_sql(inmap, stmts, config=cfg) + copy_stmt = ("\\copy ", 'sd.t1', " from '", + os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH), "' csv") + assert sql[0] == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c2_fkey" + assert sql[1] == "TRUNCATE ONLY sd.t1" + assert sql[2] == copy_stmt + assert sql[3] == "ALTER TABLE sd.t2 ADD CONSTRAINT t2_c2_fkey " \ + "FOREIGN KEY (c2) REFERENCES sd.t1 (pc1)" diff --git a/tests/dbobject/test_table.py b/tests/dbobject/test_table.py new file mode 100644 index 0000000..7aa7edc --- /dev/null +++ b/tests/dbobject/test_table.py @@ -0,0 +1,436 @@ +# -*- coding: utf-8 -*- +"""Test tables""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +COMMENT_STMT = "COMMENT ON TABLE sd.t1 IS 'Test table t1'" +CREATE_STOR_PARAMS = CREATE_STMT + \ + " WITH (fillfactor=90, autovacuum_enabled=false)" + + +class TableToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created tables""" + + def test_map_table_simple(self): + "Map a table with two columns" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]} + + def test_map_table_comment(self): + "Map a table comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'description': 'Test table t1'} + + def test_map_table_comment_quotes(self): + "Map a table comment with quotes" + stmts = [CREATE_STMT, "COMMENT ON TABLE t1 IS " + "'A \"special\" person''s table t1'"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'description': "A \"special\" person's table t1"} + + def test_map_column_comments(self): + "Map two column comments" + stmts = [CREATE_STMT, + "COMMENT ON COLUMN t1.c1 IS 'Test column c1 of t1'", + "COMMENT ON COLUMN t1.c2 IS 'Test column c2 of t1'"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', + 'description': 'Test column c1 of t1'}}, + {'c2': {'type': 'text', + 'description': 'Test column c2 of t1'}}]} + + def test_map_table_options(self): + "Map a table with options" + dbmap = self.to_map([CREATE_STOR_PARAMS]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=90", 'autovacuum_enabled=false']} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_inherit(self): + "Map a table that inherits from two other tables" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c3 integer)", + "CREATE TABLE t3 (c4 text) INHERITS (t1, t2)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'integer', 'inherited': True}}, + {'c4': {'type': 'text'}}], + 'inherits': ['t1', 't2']} + assert dbmap['schema sd']['table t3'] == expmap + + def test_map_inherit_delim(self): + "Map a table that inherits from two other tables (delim identifiers)" + stmts = [CREATE_STMT, "CREATE TABLE \"t-2\" (c3 integer)", + "CREATE TABLE \"t-3\" (c4 text) INHERITS (t1, \"t-2\")"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'integer', 'inherited': True}}, + {'c4': {'type': 'text'}}], + "inherits": ["t1", "t-2"]} + assert dbmap["schema sd"]["table t-3"] == expmap + + def test_map_unlogged_table(self): + "Map an unlogged table" + dbmap = self.to_map(["CREATE UNLOGGED TABLE t1 (c1 integer, c2 text)"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unlogged': True} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_table_within_schema(self): + "Map a schema and a table within it" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 INTEGER, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema s1'] == { + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}} + + def test_map_table_quoted(self): + "Map a schema and a table both of which need to be quoted" + stmts = ['CREATE SCHEMA "a schema"', + 'CREATE TABLE "a schema"."The.Table" ("column" SERIAL, ' + 'c2 TEXT)'] + dbmap = self.to_map(stmts) + assert dbmap['schema a schema']['table The.Table'] == { + 'columns': [{'column': { + 'type': 'integer', 'not_null': True, + 'default': + 'nextval(\'"a schema"."The.Table_column_seq"\'::regclass)'}}, + {'c2': {'type': 'text'}}]} + + def test_map_select_tables(self): + "Map two tables out of three present" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE TABLE t3 (c1 integer, c2 text)"] + dbmap = self.to_map(stmts, tables=['t2', 't1']) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + assert dbmap['schema sd']['table t2'] == expmap + assert 'table t3' not in dbmap['schema sd'] + + def test_map_table_sequence(self): + "Map sequence if owned by a table" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE SEQUENCE seq1", "ALTER SEQUENCE seq1 OWNED BY t2.c1", + "CREATE SEQUENCE seq2"] + dbmap = self.to_map(stmts, tables=['t2']) + self.db.execute_commit("DROP SEQUENCE seq1") + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + assert 'table t1' not in dbmap['schema sd'] + assert dbmap['schema sd']['table t2'] == expmap + assert 'sequence seq1' in dbmap['schema sd'] + assert 'sequence seq2' not in dbmap['schema sd'] + + def test_map_partition_range(self): + "Map a partitioned table with two partitions by range" + spec1 = "FROM ('2015-01-01', MINVALUE) TO ('2016-12-31', 5)" + stmts = ["CREATE TABLE t1 (c1 date, c2 integer, c3 text) " + "PARTITION BY RANGE (c1, c2)", + "CREATE TABLE t1a PARTITION OF t1 FOR VALUES %s" % spec1, + "CREATE TABLE t1b PARTITION OF t1 FOR VALUES " + "FROM ('2017-01-01', 11) TO ('2020-12-31', 15)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'date'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'partition_by': {'range': ['c1', 'c2']}} + assert dbmap['schema sd']['table t1'] == expmap + expmap2 = {'partition_bound_spec': spec1, 'partition_of': 't1'} + assert dbmap['schema sd']['table t1a'] == expmap2 + + def test_map_partition_list(self): + "Map a partitioned table with two partitions by list" + spec1 = "IN (1, 3, 5, 7)" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text) " + "PARTITION BY LIST (c1)", + "CREATE TABLE t1a PARTITION OF t1 FOR VALUES %s" % spec1, + "CREATE TABLE t1b PARTITION OF t1 FOR VALUES IN (2, 4, 6)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'partition_by': {'list': ['c1']}} + assert dbmap['schema sd']['table t1'] == expmap + expmap2 = {'partition_bound_spec': spec1, 'partition_of': 't1'} + assert dbmap['schema sd']['table t1a'] == expmap2 + + +class TableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table statements from input schemas""" + + def test_create_table_simple(self): + "Create a two-column table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_table_quoted_idents(self): + "Create a table needing quoted identifiers" + inmap = self.std_map() + inmap['schema sd'].update({'table order': { + 'columns': [{'primary': {'type': 'integer'}}, + {'two words': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, quote_reserved=True) + assert fix_indent(sql[0]) == 'CREATE TABLE sd."order" (' \ + '"primary" integer, "two words" text)' + + def test_bad_table_map(self): + "Error creating a table with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_missing_columns(self): + "Error creating a table with no columns" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': {'columns': []}}) + with pytest.raises(ValueError): + self.to_sql(inmap) + + def test_drop_table(self): + "Drop an existing table" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP TABLE sd.t1"] + + def test_rename_table(self): + "Rename an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t2': { + 'oldname': 't1', + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER TABLE sd.t1 RENAME TO t2"] + + def test_create_table_options(self): + "Create a table with options" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=90", "autovacuum_enabled=false"]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STOR_PARAMS + + def test_change_table_options(self): + "Change a table's storage parameters" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=70"]}}) + sql = self.to_sql(inmap, [CREATE_STOR_PARAMS]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET (fillfactor=70)," \ + " RESET (autovacuum_enabled)" + + def test_create_table_within_schema(self): + "Create a new schema and a table within it" + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + sql = self.to_sql(inmap) + expsql = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text)"] + for i in range(len(expsql)): + assert fix_indent(sql[i]) == expsql[i] + + def test_unlogged_table(self): + "Create an unlogged table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'unlogged': True}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == \ + "CREATE UNLOGGED TABLE sd.t1 (c1 integer, c2 text)" + + def test_table_owned_by_sequence(self): + "Alter a table to be owned by a table column" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}, + 'sequence seq1': { + 'cache_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}) + sql = self.to_sql(inmap, [CREATE_STMT, "CREATE SEQUENCE seq1"]) + assert sql[0] == "ALTER SEQUENCE sd.seq1 OWNED BY sd.t1.c1" + + def test_create_partitioned_tables(self): + "Create a partitioned table and two partitions" + inmap = self.std_map() + spec1 = "FROM ('2015-01-01', MINVALUE) TO ('2016-12-31', 5)" + spec2 = "FROM ('2017-01-01', 11) TO ('2020-12-31', 15)" + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'date'}}, {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'partition_by': {'range': ['c1', 'c2']}}, 'table t1a': { + 'partition_bound_spec': spec1, 'partition_of': 't1'}, + 'table t1b': {'partition_bound_spec': spec2, + 'partition_of': 't1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 date, " \ + "c2 integer, c3 text) PARTITION BY RANGE (c1, c2)" + assert fix_indent(sql[1]) == ( + "CREATE TABLE sd.t1a PARTITION OF t1 FOR VALUES %s" % spec1) + assert fix_indent(sql[2]) == ( + "CREATE TABLE sd.t1b PARTITION OF t1 FOR VALUES %s" % spec2) + + +class TableCommentToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table and column COMMENT statements""" + + def _tblmap(self): + "Return a table input map with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'description': 'Test table t1', + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + return inmap + + def test_table_with_comment(self): + "Create a table with a comment" + sql = self.to_sql(self._tblmap()) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_table(self): + "Create a comment for an existing table" + sql = self.to_sql(self._tblmap(), [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_table_comment_quotes(self): + "Create a table comment with quotes" + inmap = self._tblmap() + inmap['schema sd']['table t1']['description'] = \ + "A \"special\" person's table t1" + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS " + "'A \"special\" person''s table t1'"] + + def test_drop_table_comment(self): + "Drop a comment on an existing table" + inmap = self._tblmap() + del inmap['schema sd']['table t1']['description'] + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS NULL"] + + def test_change_table_comment(self): + "Change existing comment on a table" + inmap = self._tblmap() + inmap['schema sd']['table t1'].update( + {'description': 'Changed table t1'}) + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS 'Changed table t1'"] + + def test_create_column_comments(self): + "Create a table with column comments" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Test column c1') + inmap['schema sd']['table t1']['columns'][1]['c2'].update( + description='Test column c2') + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + assert sql[2] == "COMMENT ON COLUMN sd.t1.c1 IS 'Test column c1'" + assert sql[3] == "COMMENT ON COLUMN sd.t1.c2 IS 'Test column c2'" + + def test_add_column_comment(self): + "Add a column comment to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Test column c1') + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS 'Test column c1'" + + def test_add_column_with_comment(self): + "Add a commented column to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'].append({'c3': { + 'description': 'Test column c3', 'type': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 integer" + assert sql[1] == "COMMENT ON COLUMN sd.t1.c3 IS 'Test column c3'" + + def test_drop_column_comment(self): + "Drop a column comment on an existing table" + stmts = [CREATE_STMT, COMMENT_STMT, + "COMMENT ON COLUMN t1.c1 IS 'Test column c1'"] + sql = self.to_sql(self._tblmap(), stmts) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS NULL" + + def test_change_column_comment(self): + "Add a column comment to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Changed column c1') + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS 'Changed column c1'" + + +class TableInheritToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table inheritance statements""" + + def test_table_inheritance(self): + "Create a table that inherits from another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'numeric'}}], 'inherits': ['t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert fix_indent(sql[1]) == "CREATE TABLE sd.t2 (c3 numeric) " \ + "INHERITS (sd.t1)" + + def test_table_inherit_delim(self): + "Create a table that inherits from another (delimited identifiers)" + inmap = self.std_map() + inmap.update({'schema s-d': {'table t-1': { + 'columns': [{'c-1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + inmap['schema s-d'].update({'table t-2': { + 'columns': [{'c-1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'numeric'}}], 'inherits': ['t-1']}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA \"s-d\""]) + print(sql) + assert fix_indent(sql[0]) == "CREATE TABLE \"s-d\".\"t-1\" " \ + "(\"c-1\" integer, c2 text)" + assert fix_indent(sql[1]) == "CREATE TABLE \"s-d\".\"t-2\" " \ + "(c3 numeric) INHERITS (\"s-d\".\"t-1\")" + + def test_drop_inherited(self): + "Drop tables that inherit from others" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c3 numeric) INHERITS (t1)", + "CREATE TABLE t3 (c4 date) INHERITS (t2)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql == ["DROP TABLE sd.t3", "DROP TABLE sd.t2", + "DROP TABLE sd.t1"] diff --git a/tests/dbobject/test_tablespace.py b/tests/dbobject/test_tablespace.py new file mode 100644 index 0000000..45cbc7c --- /dev/null +++ b/tests/dbobject/test_tablespace.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Test tablespaces + +These tests require the existence of tablespaces ts1 and ts2. +They should be owned by the user running the tests or the user should +have been granted CREATE (or ALL) privileges on the tablespaces. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + + +CREATE_TABLE = "CREATE TABLE sd.t1 (c1 integer, c2 text) TABLESPACE ts1" + + +class ToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created tables""" + + def test_map_table(self): + "Map a table using a tablespace" + dbmap = self.to_map([CREATE_TABLE]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'tablespace': 'ts1'} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_primary_key(self): + "Map a table with a PRIMARY KEY using a tablespace" + dbmap = self.to_map(["CREATE TABLE t1 (c1 integer PRIMARY KEY " + "USING INDEX TABLESPACE ts1, c2 text)"]) + expmap = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], + 'tablespace': 'ts1'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index(self): + "Map an index using a tablespace" + dbmap = self.to_map(["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE UNIQUE INDEX t1_idx ON t1 (c1) " + "TABLESPACE ts1"]) + expmap = {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts1', + 'unique': True}} + assert dbmap['schema sd']['table t1']['indexes'] == expmap + + +class ToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table statements from input schemas""" + + def test_create_table(self): + "Create a table in a tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE + + def test_move_table(self): + "Move a table from one tablespace to another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'tablespace': 'ts2'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET TABLESPACE ts2" + + def test_create_primary_key(self): + "Create a table with a PRIMARY KEY in a different tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], + 'tablespace': 'ts2'}}, + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 " \ + "(c1 integer NOT NULL, c2 text) TABLESPACE ts1" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_pkey PRIMARY KEY (c1) USING INDEX TABLESPACE ts2" + + def test_create_index(self): + "Create an index using a tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts2', + 'unique': True}}, + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert fix_indent(sql[0]) == "CREATE UNIQUE INDEX t1_idx " \ + "ON sd.t1 (c1) TABLESPACE ts2" + + def test_move_index(self): + "Move a index from one tablespace to another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts2'}}}}) + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE INDEX t1_idx ON t1 (c1) TABLESPACE ts1"] + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER INDEX sd.t1_idx SET TABLESPACE ts2" diff --git a/tests/dbobject/test_textsearch.py b/tests/dbobject/test_textsearch.py new file mode 100644 index 0000000..3093cef --- /dev/null +++ b/tests/dbobject/test_textsearch.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +"""Test text search objects""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TSC_STMT = "CREATE TEXT SEARCH CONFIGURATION sd.tsc1 (PARSER = tsp1)" +CREATE_TSD_STMT = "CREATE TEXT SEARCH DICTIONARY sd.tsd1 (TEMPLATE = simple, "\ + "stopwords = 'english')" +CREATE_TSP_STMT = "CREATE TEXT SEARCH PARSER sd.tsp1 (START = prsd_start, " \ + "GETTOKEN = prsd_nexttoken, END = prsd_end, LEXTYPES = prsd_lextype, " \ + "HEADLINE = prsd_headline)" +CREATE_TST_STMT = "CREATE TEXT SEARCH TEMPLATE sd.tst1 (INIT = dsimple_init, "\ + "LEXIZE = dsimple_lexize)" +COMMENT_TSC_STMT = "COMMENT ON TEXT SEARCH CONFIGURATION sd.tsc1 IS " \ + "'Test configuration tsc1'" +COMMENT_TSD_STMT = "COMMENT ON TEXT SEARCH DICTIONARY sd.tsd1 IS " \ + "'Test dictionary tsd1'" +COMMENT_TSP_STMT = "COMMENT ON TEXT SEARCH PARSER sd.tsp1 IS " \ + "'Test parser tsp1'" +COMMENT_TST_STMT = "COMMENT ON TEXT SEARCH TEMPLATE sd.tst1 IS " \ + "'Test template tst1'" + + +class TextSearchConfigToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search configurations""" + + superuser = True + + def test_map_ts_config(self): + "Map an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'] == { + 'parser': 'tsp1'} + + def test_map_cross_schema_ts_config(self): + "Map a text search config with parser in different schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TEXT SEARCH PARSER s1.tsp1 " + "(START = prsd_start, GETTOKEN = prsd_nexttoken, " + "END = prsd_end, LEXTYPES = prsd_lextype)", + "CREATE TEXT SEARCH CONFIGURATION tsc1 (PARSER = s1.tsp1)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'] == { + 'parser': 's1.tsp1'} + + def test_map_ts_config_comment(self): + "Map a text search configuration with a comment" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT, COMMENT_TSC_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'][ + 'description'] == 'Test configuration tsc1' + + +class TextSearchConfigToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search configurations""" + + def test_create_ts_config(self): + "Create a text search configuration that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}, 'text search configuration tsc1': { + 'parser': 'tsp1'}}) + sql = self.to_sql(inmap, [CREATE_TSP_STMT]) + assert fix_indent(sql[0]) == CREATE_TSC_STMT + + def test_create_ts_config_in_schema(self): + "Create a text search config with parser in non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype'}}}) + inmap['schema sd'].update({'text search configuration tsc1': { + 'parser': 's1.tsp1'}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE TEXT SEARCH PARSER s1.tsp1 " \ + "(START = prsd_start, GETTOKEN = prsd_nexttoken, " \ + "END = prsd_end, LEXTYPES = prsd_lextype)" + assert fix_indent(sql[1]) == \ + "CREATE TEXT SEARCH CONFIGURATION sd.tsc1 (PARSER = s1.tsp1)" + + def test_bad_map_ts_config_(self): + "Error creating a text search configuration with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsc1': {'parser': 'tsp1'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_config(self): + "Drop an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP TEXT SEARCH CONFIGURATION sd.tsc1" + assert sql[1] == "DROP TEXT SEARCH PARSER sd.tsp1" + + def test_comment_on_ts_config(self): + "Create a comment for an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'text search configuration tsc1': { + 'parser': 'tsp1', 'description': "Test configuration tsc1"}, + 'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_TSC_STMT] + + +class TextSearchDictToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search dictionaries""" + + def test_map_ts_dict(self): + "Map an existing text search dictionary" + dbmap = self.to_map([CREATE_TSD_STMT]) + assert dbmap['schema sd']['text search dictionary tsd1'] == { + 'template': 'simple', 'options': "stopwords = 'english'"} + + def test_map_ts_dict_comment(self): + "Map a text search dictionary with a comment" + stmts = [CREATE_TSD_STMT, COMMENT_TSD_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search dictionary tsd1'][ + 'description'], 'Test dictionary tsd1' + + +class TextSearchDictToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search dictionaries""" + + def test_create_ts_dict(self): + "Create a text search dictionary that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search dictionary tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TSD_STMT + + def test_bad_map_ts_dict(self): + "Error creating a text search dictionary with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'"}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_dict(self): + "Drop an existing text search dictionary" + sql = self.to_sql(self.std_map(), [CREATE_TSD_STMT]) + assert sql == ["DROP TEXT SEARCH DICTIONARY sd.tsd1"] + + def test_comment_on_ts_dict(self): + "Create a comment for an existing text search dictionary" + inmap = self.std_map() + inmap['schema sd'].update({'text search dictionary tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'", + 'description': "Test dictionary tsd1"}}) + sql = self.to_sql(inmap, [CREATE_TSD_STMT]) + assert sql == [COMMENT_TSD_STMT] + + +class TextSearchParserToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search parsers""" + + superuser = True + + def test_map_ts_parser(self): + "Map an existing text search parser" + stmts = [CREATE_TSP_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search parser tsp1'] == { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'} + + def test_map_ts_parser_comment(self): + "Map a text search parser with a comment" + stmts = [CREATE_TSP_STMT, COMMENT_TSP_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search parser tsp1'][ + 'description'] == 'Test parser tsp1' + + +class TextSearchParserToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search parsers""" + + def test_create_ts_parser(self): + "Create a text search parser that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TSP_STMT + + def test_bad_map_ts_parser(self): + "Error creating a text search parser with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_parser(self): + "Drop an existing text search parser" + sql = self.to_sql(self.std_map(), [CREATE_TSP_STMT], superuser=True) + assert sql == ["DROP TEXT SEARCH PARSER sd.tsp1"] + + def test_comment_on_ts_parser(self): + "Create a comment for an existing text search parser" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline', 'description': "Test parser tsp1"}}) + sql = self.to_sql(inmap, [CREATE_TSP_STMT], superuser=True) + assert sql == [COMMENT_TSP_STMT] + + +class TextSearchTemplateToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search templates""" + + superuser = True + + def test_map_ts_template(self): + "Map an existing text search template" + dbmap = self.to_map([CREATE_TST_STMT]) + assert dbmap['schema sd']['text search template tst1'] == { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'} + + def test_map_ts_template_comment(self): + "Map a text search template with a comment" + stmts = [CREATE_TST_STMT, COMMENT_TST_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search template tst1'][ + 'description'], 'Test template tst1' + + +class TextSearchTemplateToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search templates""" + + def test_create_ts_template(self): + "Create a text search template that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search template tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'}}) + sql = self.to_sql(inmap, superuser=True) + assert fix_indent(sql[0]) == CREATE_TST_STMT + + def test_bad_map_ts_template(self): + "Error creating a text search template with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_template(self): + "Drop an existing text search template" + sql = self.to_sql(self.std_map(), [CREATE_TST_STMT], superuser=True) + assert sql == ["DROP TEXT SEARCH TEMPLATE sd.tst1"] + + def test_comment_on_ts_template(self): + "Create a comment for an existing text search template" + inmap = self.std_map() + inmap['schema sd'].update({'text search template tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize', + 'description': "Test template tst1"}}) + sql = self.to_sql(inmap, [CREATE_TST_STMT], superuser=True) + assert sql == [COMMENT_TST_STMT] diff --git a/tests/dbobject/test_trigger.py b/tests/dbobject/test_trigger.py new file mode 100644 index 0000000..056a7c4 --- /dev/null +++ b/tests/dbobject/test_trigger.py @@ -0,0 +1,488 @@ +# -*- coding: utf-8 -*- +"""Test triggers""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +FUNC_SRC = "BEGIN NEW.c3 := CURRENT_DATE; RETURN NEW; END" +FUNC_INSTEAD_SRC = "BEGIN INSERT INTO t1 VALUES (NEW.c1, NEW.c2, now()); " \ + "RETURN NULL; END" +FUNC_REF_NEW_SRC = "BEGIN SELECT SUM(c2) FROM ins; RETURN NULL; END" +FUNC_REF_NEW_OLD_SRC = "BEGIN SELECT SUM(c2) FROM del; " \ + "SELECT SUM(c2) FROM ins; RETURN NULL; END" +CREATE_TABLE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text, " \ + "c3 date)" +CREATE_TABLE_STMT2 = "CREATE TABLE t1 (c1 integer, c2 text, " \ + "c3 text, tsidx tsvector)" +CREATE_TABLE_STMT3 = "CREATE TABLE sd.t1 (c1 integer, c2 money)" +CREATE_FUNC_STMT = "CREATE FUNCTION sd.f1() RETURNS trigger LANGUAGE plpgsql" \ + " AS $_$%s$_$" % FUNC_SRC +CREATE_STMT = "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON sd.t1 " \ + "FOR EACH ROW EXECUTE PROCEDURE sd.f1()" +COMMENT_STMT = "COMMENT ON TRIGGER tr1 ON sd.t1 IS 'Test trigger tr1'" + + +class TriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing triggers""" + + def test_map_trigger1(self): + "Map a simple trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}} + + def test_map_trigger2(self): + "Map another simple trigger with different attributes" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER DELETE OR TRUNCATE ON t1 " + "EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['delete', 'truncate'], + 'level': 'statement', 'procedure': 'sd.f1'}} + + def test_map_trigger_update_cols(self): + "Map trigger with UPDATE OF columns" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER INSERT OR UPDATE OF c1, c2 ON t1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['insert', 'update'], + 'columns': ['c1', 'c2'], 'level': 'row', + 'procedure': 'sd.f1'}} + + def test_map_trigger_conditional(self): + "Map trigger with a WHEN qualification" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER UPDATE ON t1 FOR EACH ROW " + "WHEN (OLD.c2 IS DISTINCT FROM NEW.c2) " + "EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'procedure': 'sd.f1', + 'condition': '(old.c2 IS DISTINCT FROM new.c2)'}} + + def test_map_trigger_instead(self): + "Map an INSTEAD OF trigger" + stmts = [CREATE_TABLE_STMT, "CREATE VIEW v1 AS SELECT c1, c2 FROM t1", + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_INSTEAD_SRC, + "CREATE TRIGGER tr1 INSTEAD OF INSERT ON v1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['view v1']['triggers'] == { + 'tr1': {'timing': 'instead of', 'events': ['insert'], + 'level': 'row', 'procedure': 'sd.f1'}} + + def test_map_tsvector_trigger(self): + "Map a text search (tsvector) trigger" + stmts = [ + CREATE_TABLE_STMT2, + "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON sd.t1 " + "FOR EACH ROW EXECUTE PROCEDURE " + "tsvector_update_trigger('tsidx', 'pg_catalog.english', 'c2')"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', + 'procedure': {'name': 'tsvector_update_trigger', + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2'"}}} + + def test_map_trigger_function_distinct_schemas(self): + "Map a trigger in a non-default schema with function in different one" + stmts = ["CREATE SCHEMA s1", "CREATE TABLE s1.t1 (c1 integer, " + "c2 text, c3 date)", "CREATE SCHEMA s2", + "CREATE FUNCTION s2.f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_SRC, + "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON s1.t1 " + "FOR EACH ROW EXECUTE PROCEDURE s2.f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema s1']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 's2.f1'}} + + def test_map_trigger_referencing_new(self): + "Map a trigger that uses REFERENCING NEW TABLE" + stmts = [CREATE_TABLE_STMT3, + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_REF_NEW_SRC, + "CREATE TRIGGER tr1 AFTER INSERT ON t1 " + "REFERENCING NEW TABLE AS ins " + "FOR EACH STATEMENT EXECUTE FUNCTION f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['insert'], + 'level': 'statement', 'procedure': 'sd.f1', + 'referencing_new': 'ins'}} + + def test_map_trigger_referencing_new_old(self): + "Map a trigger that uses REFERENCING NEW and OLD TABLE" + stmts = [CREATE_TABLE_STMT3, + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_REF_NEW_OLD_SRC, + "CREATE TRIGGER tr1 AFTER UPDATE ON t1 " + "REFERENCING NEW TABLE AS ins OLD TABLE AS del " + "FOR EACH ROW EXECUTE FUNCTION f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'procedure': 'sd.f1', + 'referencing_new': 'ins', 'referencing_old': 'del'}} + + def test_map_trigger_comment(self): + "Map a trigger comment" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers']['tr1'][ + 'description'] == 'Test trigger tr1' + + +class ConstraintTriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing constraint triggers""" + + def test_map_trigger(self): + "Map a simple constraint trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE CONSTRAINT TRIGGER tr1 AFTER INSERT OR UPDATE ON t1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'constraint': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}} + + def test_map_trigger_deferrable(self): + "Map a deferrable, initially deferred constraint trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE CONSTRAINT TRIGGER tr1 AFTER INSERT OR UPDATE ON t1 " + "DEFERRABLE INITIALLY DEFERRED " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'constraint': True, 'deferrable': True, + 'initially_deferred': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}} + + +class TriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_trigger1(self): + "Create a simple trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == CREATE_STMT + + def test_create_trigger2(self): + "Create another simple trigger with" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'after', + 'events': ['delete', 'truncate'], + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 AFTER DELETE OR " \ + "TRUNCATE ON sd.t1 FOR EACH STATEMENT EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_update_cols(self): + "Create a trigger with UPDATE OF columns" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'before', 'events': [ + 'insert', 'update'], 'columns': ['c1', 'c2'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE INSERT OR " \ + "UPDATE OF c1, c2 ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_conditional(self): + "Create a trigger with a WHEN qualification" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'before', 'events': [ + 'update'], 'level': 'row', 'procedure': 'sd.f1', + 'condition': '(old.c2 IS DISTINCT FROM new.c2)'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE UPDATE " \ + "ON sd.t1 FOR EACH ROW WHEN ((old.c2 IS DISTINCT FROM new.c2)) " \ + "EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_instead(self): + "Create an INSTEAD OF trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', + 'source': FUNC_INSTEAD_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}, + 'view v1': {'definition': "SELECT c1, c2 FROM t1", + 'triggers': {'tr1': {'timing': 'instead of', + 'events': ['insert'], + 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + cr1, cr2 = (1, 2) if 'VIEW' in sql[1] else (2, 1) + assert fix_indent(sql[cr1]) == \ + "CREATE VIEW sd.v1 AS SELECT c1, c2 FROM t1" + assert fix_indent(sql[cr2]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_INSTEAD_SRC + assert fix_indent(sql[3]) == "CREATE TRIGGER tr1 INSTEAD OF INSERT " \ + "ON sd.v1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_add_tsvector_trigger(self): + "Add a text search (tsvector) trigger" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}, + {'tsidx': {'type': 'tsvector'}}], + 'triggers': {'t1_tsidx_update': { + 'timing': 'before', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': {'name': 'tsvector_update_trigger', + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2'"}}}}}) + sql = self.to_sql(inmap, [CREATE_TABLE_STMT2]) + assert fix_indent(sql[0]) == "CREATE TRIGGER t1_tsidx_update BEFORE" \ + " INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE " \ + "tsvector_update_trigger('tsidx', 'pg_catalog.english', 'c2')" + + def test_change_tsvector_trigger(self): + "Change a text search (tsvector) trigger" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}, + {'tsidx': {'type': 'tsvector'}}], + 'triggers': {'t1_tsidx_update': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', + 'procedure': {'name': "tsvector_update_trigger", + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2', 'c3'"}}}}}) + stmts = [CREATE_TABLE_STMT2, + "CREATE TRIGGER t1_tsidx_update BEFORE INSERT OR UPDATE ON " + "t1 FOR EACH ROW EXECUTE PROCEDURE tsvector_update_trigger" + "('tsidx', 'pg_catalog.english', 'c2')"] + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP TRIGGER t1_tsidx_update ON sd.t1" + assert fix_indent(sql[1]) == "CREATE TRIGGER t1_tsidx_update BEFORE" \ + " INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE " \ + "tsvector_update_trigger('tsidx', 'pg_catalog.english', " \ + "'c2', 'c3')" + + def test_create_trigger_function_distinct_schemas(self): + "Create a trigger in non-default schema with function in different one" + inmap = self.std_map(plpgsql_installed=True) + inmap.update({'schema s2': {'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}, + 'schema s1': { + 'table t1': { + 'columns': [ + {'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', + 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 's2.f1'}}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1", "CREATE SCHEMA s2"]) + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE INSERT OR " \ + "UPDATE ON s1.t1 FOR EACH ROW EXECUTE PROCEDURE s2.f1()" + + def test_create_trigger_referencing(self): + "Create a trigger that uses REFERENCING OLD and NEW" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', + 'source': FUNC_REF_NEW_OLD_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'money'}}], + 'triggers': {'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'referencing_new': 'ins', + 'referencing_old': 'del', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT3 + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_REF_NEW_OLD_SRC + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 AFTER UPDATE " \ + "ON sd.t1 REFERENCING NEW TABLE AS ins OLD TABLE AS del " \ + "FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_drop_trigger(self): + "Drop an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP TRIGGER tr1 ON sd.t1"] + + def test_drop_trigger_table(self): + "Drop an existing trigger and the related table" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP TRIGGER tr1 ON sd.t1" + assert sql[1] == "DROP TABLE sd.t1" + + def test_trigger_with_comment(self): + "Create a trigger with a comment" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Test trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == CREATE_STMT + assert sql[3] == COMMENT_STMT + + def test_comment_on_trigger(self): + "Create a comment on an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Test trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_trigger_comment(self): + "Drop a comment on an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON TRIGGER tr1 ON sd.t1 IS NULL"] + + def test_change_trigger_comment(self): + "Change existing comment on a trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Changed trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [ + "COMMENT ON TRIGGER tr1 ON sd.t1 IS 'Changed trigger tr1'"] + + +class ConstraintTriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_trigger(self): + "Create a constraint trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'constraint': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE CONSTRAINT TRIGGER tr1 AFTER " \ + "INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_deferrable(self): + "Create a deferrable constraint trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'constraint': True, 'deferrable': True, + 'initially_deferred': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE CONSTRAINT TRIGGER tr1 " \ + "AFTER INSERT OR UPDATE ON sd.t1 DEFERRABLE INITIALLY " \ + "DEFERRED FOR EACH ROW EXECUTE PROCEDURE sd.f1()" diff --git a/tests/dbobject/test_type.py b/tests/dbobject/test_type.py new file mode 100644 index 0000000..b7a20e1 --- /dev/null +++ b/tests/dbobject/test_type.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +"""Test enums and other types""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_COMPOSITE_STMT = "CREATE TYPE sd.t1 AS " \ + "(x integer, y integer, z integer)" +CREATE_ENUM_STMT = "CREATE TYPE sd.t1 AS ENUM ('red', 'green', 'blue')" +CREATE_SHELL_STMT = "CREATE TYPE sd.t1" +CREATE_RANGE_STMT = "CREATE TYPE sd.t1 AS RANGE (SUBTYPE = smallint)" +CREATE_FUNC_IN = "CREATE FUNCTION sd.t1textin(cstring) RETURNS t1 " \ + "LANGUAGE internal IMMUTABLE STRICT AS $$textin$$" +CREATE_FUNC_OUT = "CREATE FUNCTION sd.t1textout(sd.t1) RETURNS cstring " \ + "LANGUAGE internal IMMUTABLE STRICT AS $$textout$$" +CREATE_TYPE_STMT = "CREATE TYPE t1 (INPUT = t1textin, OUTPUT = t1textout)" +COMMENT_STMT = "COMMENT ON TYPE t1 IS 'Test type t1'" + + +class CompositeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created composite types""" + + def test_composite(self): + "Map a composite type" + dbmap = self.to_map([CREATE_COMPOSITE_STMT]) + assert dbmap['schema sd']['type t1'] == { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]} + + def test_dropped_attribute(self): + "Map a composite type which has a dropped attribute" + stmts = [CREATE_COMPOSITE_STMT, "ALTER TYPE t1 DROP ATTRIBUTE y"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]} + + +class CompositeToSqlTestCase(InputMapToSqlTestCase): + """Test creation and modification of composite types""" + + def test_create_composite(self): + "Create a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_COMPOSITE_STMT + + def test_drop_composite(self): + "Drop an existing composite" + sql = self.to_sql(self.std_map(), [CREATE_COMPOSITE_STMT]) + assert sql == ["DROP TYPE sd.t1"] + + def test_rename_composite(self): + "Rename an existing composite" + inmap = self.std_map() + inmap['schema sd'].update({'type t2': { + 'oldname': 't1', + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert sql == ["ALTER TYPE sd.t1 RENAME TO t2"] + + def test_add_attribute(self): + "Add an attribute to a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, ["CREATE TYPE t1 AS (x integer, y integer)"]) + assert fix_indent(sql[0]) == "ALTER TYPE sd.t1 ADD ATTRIBUTE z integer" + + def test_drop_attribute(self): + "Drop an attribute from a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert fix_indent(sql[0]) == "ALTER TYPE sd.t1 DROP ATTRIBUTE y" + + def test_drop_attribute_schema(self): + "Drop an attribute from a composite type within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}}) + sql = self.to_sql(inmap, [ + "CREATE SCHEMA s1", + "CREATE TYPE s1.t1 AS (x integer, y integer, z integer)"]) + assert fix_indent(sql[0]) == "ALTER TYPE s1.t1 DROP ATTRIBUTE y" + + def test_rename_attribute(self): + "Rename an attribute of a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y1': {'type': 'integer', 'oldname': 'y'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert fix_indent(sql[0]) == \ + "ALTER TYPE sd.t1 RENAME ATTRIBUTE y TO y1" + + +class EnumToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created enum types""" + + def test_enum(self): + "Map an enum" + dbmap = self.to_map([CREATE_ENUM_STMT]) + assert dbmap['schema sd']['type t1'] == { + 'labels': ['red', 'green', 'blue']} + + +class EnumToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input enums""" + + def test_create_enum(self): + "Create an enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'labels': ['red', 'green', 'blue']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_ENUM_STMT + + def test_drop_enum(self): + "Drop an existing enum" + sql = self.to_sql(self.std_map(), [CREATE_ENUM_STMT]) + assert sql == ["DROP TYPE sd.t1"] + + def test_change_enum(self): + "Change an existing enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'labels': ['red', 'yellow', 'blue']}}) + sql = self.to_sql(inmap, [CREATE_ENUM_STMT]) + assert sql[0] == "DROP TYPE sd.t1" + assert fix_indent(sql[1]) == \ + "CREATE TYPE sd.t1 AS ENUM ('red', 'yellow', 'blue')" + + def test_rename_enum(self): + "Rename an existing enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t2': { + 'oldname': 't1', 'labels': ['red', 'green', 'blue']}}) + sql = self.to_sql(inmap, [CREATE_ENUM_STMT]) + assert sql == ["ALTER TYPE sd.t1 RENAME TO t2"] + + +class BaseTypeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created base type types""" + + superuser = True + + def test_base_type(self): + "Map a base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + CREATE_TYPE_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain', 'category': 'U'} + + def test_base_type_category(self): + "Map a base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + "CREATE TYPE t1 (INPUT = t1textin, OUTPUT = t1textout, " + "CATEGORY = 'S')"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain', 'category': 'S'} + + +class BaseTypeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input base types""" + + def test_create_base_type(self): + "Create a base type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain'}, 'function t1textin(cstring)': { + 'language': 'internal', 'returns': 't1', 'strict': True, + 'volatility': 'immutable', 'source': 'textin'}, + 'function t1textout(sd.t1)': { + 'language': 'internal', 'returns': 'cstring', + 'strict': True, 'volatility': 'immutable', + 'source': 'textout'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_SHELL_STMT + assert fix_indent(sql[1]) == CREATE_FUNC_IN + assert fix_indent(sql[2]) == CREATE_FUNC_OUT + assert fix_indent(sql[3]) == "CREATE TYPE sd.t1 (INPUT = t1textin, " \ + "OUTPUT = t1textout, INTERNALLENGTH = variable, " \ + "ALIGNMENT = int4, STORAGE = plain)" + + def test_drop_type(self): + "Drop an existing base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + CREATE_TYPE_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql == ["DROP TYPE sd.t1 CASCADE"] + + +class RangeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created RANGE types""" + + def test_range_simple(self): + "Map a simple range type" + dbmap = self.to_map([CREATE_RANGE_STMT]) + assert dbmap['schema sd']['type t1'] == {'subtype': 'int2'} + + def test_range_subtypediff(self): + "Map a range type with a subtype difference function" + stmts = ["CREATE TYPE t1 AS RANGE (SUBTYPE = float8, " + "SUBTYPE_DIFF = float8mi)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'subtype': 'float8', 'subtype_diff': 'float8mi'} + + +class RangeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input range types""" + + def test_create_range_simple(self): + "Create a range type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': {'subtype': 'smallint'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_RANGE_STMT + + def test_create_range_subtype_diff(self): + "Create a range with a subtype diff function" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'subtype': 'float8', 'subtype_diff': 'float8mi'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == ( + "CREATE TYPE sd.t1 AS RANGE (SUBTYPE = float8, " + "SUBTYPE_DIFF = float8mi)") diff --git a/tests/dbobject/test_view.py b/tests/dbobject/test_view.py new file mode 100644 index 0000000..4ea97a0 --- /dev/null +++ b/tests/dbobject/test_view.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +"""Test views""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE VIEW sd.v1 AS SELECT now()::date AS today" +CREATE_TBL = "CREATE TABLE sd.t1 (c1 integer, c2 text, c3 integer)" +CREATE_STMT2 = "CREATE VIEW sd.v1 AS SELECT c1, c3 * 2 AS c2 FROM t1" +COMMENT_STMT = "COMMENT ON VIEW sd.v1 IS 'Test view v1'" +VIEW_DEFN = " SELECT now()::date AS today;" + + +class ViewToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created views""" + + def test_map_view_no_table(self): + "Map a created view without a table dependency" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_table(self): + "Map a created view with a table dependency" + stmts = [CREATE_TBL, CREATE_STMT2] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}], + 'depends_on': ['table t1'], + 'definition': " SELECT t1.c1," + "\n t1.c3 * 2 AS c2\n FROM sd.t1;"} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_columns(self): + "Map a complex view's columns in addition to its definition" + stmts = ["CREATE TABLE t1 (c1 INTEGER UNIQUE)", + "CREATE TABLE t2 (c2 INTEGER PRIMARY KEY REFERENCES t1(c1))", + "CREATE VIEW v1 AS SELECT (ROW(t1.*)::t2).*, t1, 5 AS const " + "FROM t1"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c2': {'type': 'integer'}}, + {'t1': {'type': 'sd.t1'}}, + {'const': {'type': 'integer'}}], + 'definition': " SELECT (ROW(t1.c1)::sd.t2).c2 AS c2," + "\n t1.*::sd.t1 AS t1,\n 5 AS const\n FROM sd.t1;", + 'depends_on': ['table t1']} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_comment(self): + "Map a view with a comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['view v1']['description'] == \ + 'Test view v1' + + +class ViewToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input views""" + + def test_create_view_no_table(self): + "Create a view with no table dependency" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': {'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_view(self): + "Create a view" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'view v1': { + 'definition': "SELECT c1, c3 * 2 AS c2 FROM t1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TBL + assert fix_indent(sql[1]) == CREATE_STMT2 + + def test_create_view_in_schema(self): + "Create a view within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'view v1': {'definition': VIEW_DEFN}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == \ + "CREATE VIEW s1.v1 AS SELECT now()::date AS today" + + def test_bad_view_map(self): + "Error creating a view with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'v1': {'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_view_no_table(self): + "Drop an existing view without a table dependency" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP VIEW sd.v1"] + + def test_drop_view(self): + "Drop an existing view with table dependencies" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT)", + "CREATE TABLE t2 (c1 INTEGER, c3 TEXT)", + "CREATE VIEW v1 AS SELECT t1.c1, c2, c3 " + "FROM t1 JOIN t2 ON (t1.c1 = t2.c1)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP VIEW sd.v1" + # can't control which table will be dropped first + drt1 = 1 + drt2 = 2 + if 't1' in sql[2]: + drt1 = 2 + drt2 = 1 + assert sql[drt1] == "DROP TABLE sd.t1" + assert sql[drt2] == "DROP TABLE sd.t2" + + def test_rename_view(self): + "Rename an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v2': { + 'oldname': 'v1', 'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER VIEW sd.v1 RENAME TO v2"] + + def test_bad_rename_view(self): + "Error renaming a non-existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v2': { + 'oldname': 'v3', 'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_change_view_defn(self): + "Change view definition" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT 'now'::text::date AS today;"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "CREATE OR REPLACE VIEW sd.v1 AS " \ + "SELECT 'now'::text::date AS today" + + def test_change_column_name(self): + "Attempt rename view column name (disallowed)" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'todays_date': {'type': 'date'}}], + 'definition': " SELECT now()::date AS todays_date;"}}) + with pytest.raises(KeyError): + sql = self.to_sql(inmap, [CREATE_STMT]) + + def test_change_column_type(self): + "Change view column type to different type (disallowed)" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'numeric'}}], + 'definition': " SELECT c1, c3 * 2.0 AS c2 FROM t1;"}}) + with pytest.raises(TypeError): + sql = self.to_sql(inmap, [CREATE_TBL, CREATE_STMT2]) + + def test_view_depend_pk(self): + "Create a view that depends on a primary key. See issue #72" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'view v1': { + 'definition': " SELECT t1.c1,\n t1.c2\n FROM t1\n " + "GROUP BY t1.c1;", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_pkey PRIMARY KEY (c1)" + assert fix_indent(sql[2]) == "CREATE VIEW sd.v1 AS SELECT t1.c1, " \ + "t1.c2 FROM t1 GROUP BY t1.c1" + + def test_view_with_comment(self): + "Create a view with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'definition': VIEW_DEFN, 'description': "Test view v1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_view(self): + "Create a comment for an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN, 'description': "Test view v1"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_view_comment(self): + "Drop the comment on an existing view" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON VIEW sd.v1 IS NULL"] + + def test_change_view_comment(self): + "Change existing comment on a view" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN, 'description': "Changed view v1"}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON VIEW sd.v1 IS 'Changed view v1'"] diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 0000000..5c20503 --- /dev/null +++ b/tests/functional/__init__.py @@ -0,0 +1 @@ +"""Pyrseas functional tests""" diff --git a/tests/functional/autodoc-schema.sql b/tests/functional/autodoc-schema.sql new file mode 100644 index 0000000..80f31b9 --- /dev/null +++ b/tests/functional/autodoc-schema.sql @@ -0,0 +1,125 @@ +-- +-- $Id: regressdatabase.sql,v 1.2 2006/02/13 01:15:56 rbt Exp $ +-- + +BEGIN; +-- +-- Foreign key'd structure, check constraints, primary keys +-- and duplicate table names in different schemas +-- +CREATE SCHEMA product + CREATE TABLE product + ( product_id SERIAL PRIMARY KEY + , product_code text NOT NULL UNIQUE + CHECK(product_code = upper(product_code)) + , product_description text + ); + +CREATE SCHEMA store + CREATE TABLE store + ( store_id SERIAL PRIMARY KEY + , store_code text NOT NULL UNIQUE + CHECK(store_code = upper(store_code)) + , store_description text + ) + + CREATE TABLE inventory + ( store_id integer REFERENCES store + ON UPDATE CASCADE ON DELETE RESTRICT + , product_id integer REFERENCES product.product + ON UPDATE CASCADE ON DELETE RESTRICT + , PRIMARY KEY(store_id, product_id) + , quantity integer NOT NULL CHECK(quantity > 0) + ); + +-- +-- Another schema with +-- +CREATE SCHEMA warehouse + CREATE TABLE warehouse + ( warehouse_id SERIAL PRIMARY KEY + , warehouse_code text NOT NULL UNIQUE + CHECK(warehouse_code = upper(warehouse_code)) + , warehouse_manager text NOT NULL + , warehouse_supervisor text UNIQUE + , warehouse_description text + , CHECK (upper(warehouse_manager) != upper(warehouse_supervisor)) + ) + CREATE TABLE inventory + ( warehouse_id integer REFERENCES warehouse + ON UPDATE CASCADE + ON DELETE RESTRICT + , product_id integer REFERENCES product.product + ON UPDATE CASCADE + ON DELETE RESTRICT + , PRIMARY KEY(warehouse_id, product_id) + , quantity integer NOT NULL + CHECK(quantity > 0) + ) + CREATE VIEW products AS + SELECT DISTINCT product.* + FROM inventory + JOIN product.product USING (product_id); + +-- Sample index +CREATE INDEX quantity_index ON warehouse.inventory (quantity); + +-- +-- Simple text comments +-- +--COMMENT ON DATABASE IS +--'This database has been created for the purpose of simple +-- tests on PostgreSQL Autodoc.'; + +COMMENT ON SCHEMA product IS +'This schema stores a list of products and information + about the product'; + +COMMENT ON SCHEMA warehouse IS +'A list of warehouses and information on warehouses'; + +COMMENT ON TABLE warehouse.inventory IS +'Warehouse inventory'; + +COMMENT ON TABLE store.inventory IS +'Store inventory'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_code IS +'Internal code which represents warehouses for + invoice purposes'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_supervisor IS +'Supervisors name for a warehouse when one + has been assigned. The same supervisor may not + be assigned to more than one warehouse, per company + policy XYZ.'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_manager IS +'Name of Warehouse Manager'; + +-- +-- A few simple functions +-- +CREATE FUNCTION product.worker(integer, integer) RETURNS integer AS +'SELECT $1 + $1;' LANGUAGE sql; + +CREATE FUNCTION warehouse.worker(integer, integer) RETURNS integer AS +'SELECT $1 * $1;' LANGUAGE sql; + +COMMENT ON FUNCTION product.worker(integer, integer) IS +'Worker function appropriate for products'; + +COMMENT ON FUNCTION warehouse.worker(integer, integer) IS +'Worker function appropriate for warehouses.'; + + +-- +-- Inheritance +-- +CREATE SCHEMA inherit + CREATE TABLE taba (cola integer) + CREATE TABLE tabb (colb integer) inherits(taba) + CREATE TABLE tab1 (col1 integer) + CREATE TABLE tab1b (col1b integer) inherits(tab1, tabb); + +COMMIT; diff --git a/tests/functional/film-schema-0.1.sql b/tests/functional/film-schema-0.1.sql new file mode 100644 index 0000000..11ffa4c --- /dev/null +++ b/tests/functional/film-schema-0.1.sql @@ -0,0 +1,5 @@ +CREATE TABLE film ( + id INTEGER NOT NULL PRIMARY KEY, + title VARCHAR(32) NOT NULL, + release_year INTEGER NOT NULL CHECK (release_year >= 1888) +); diff --git a/tests/functional/film-schema-0.2.sql b/tests/functional/film-schema-0.2.sql new file mode 100644 index 0000000..96efc13 --- /dev/null +++ b/tests/functional/film-schema-0.2.sql @@ -0,0 +1,3 @@ +ALTER TABLE film ADD COLUMN length SMALLINT + CHECK (length > 0 AND length < 10000); +ALTER TABLE film ADD COLUMN rating CHAR(5); diff --git a/tests/functional/film-schema-0.3a.sql b/tests/functional/film-schema-0.3a.sql new file mode 100644 index 0000000..90cf545 --- /dev/null +++ b/tests/functional/film-schema-0.3a.sql @@ -0,0 +1,8 @@ +CREATE TABLE language ( + language_id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(20) NOT NULL, + last_update TIMESTAMP WITH TIME ZONE NOT NULL +); +ALTER TABLE film ADD COLUMN language_id INTEGER NOT NULL; +ALTER TABLE film ADD FOREIGN KEY (language_id) + REFERENCES language (language_id); diff --git a/tests/functional/film-schema-0.3b.sql b/tests/functional/film-schema-0.3b.sql new file mode 100644 index 0000000..e02dfc3 --- /dev/null +++ b/tests/functional/film-schema-0.3b.sql @@ -0,0 +1,34 @@ +CREATE TABLE genre ( + genre_id INTEGER PRIMARY KEY, + name VARCHAR(25) NOT NULL +); +INSERT INTO genre VALUES +(1, 'Action'), +(2, 'Adventure'), +(3, 'Animation'), +(4, 'Biography'), +(5, 'Comedy'), +(6, 'Crime'), +(7, 'Documentary'), +(8, 'Drama'), +(9, 'Family'), +(10, 'Fantasy'), +(11, 'Film-Noir'), +(12, 'History'), +(13, 'Horror'), +(14, 'Music'), +(15, 'Musical'), +(16, 'Mystery'), +(17, 'Romance'), +(18, 'Sci-Fi'), +(19, 'Sport'), +(20, 'Thriller'), +(21, 'War'), +(22, 'Western'); +CREATE TABLE film_genre ( + film_id INTEGER NOT NULL REFERENCES film (id), + genre_id INTEGER NOT NULL + REFERENCES genre (genre_id), + last_update TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY (film_id, genre_id) +); diff --git a/tests/functional/pagila-schema.sql b/tests/functional/pagila-schema.sql new file mode 100644 index 0000000..82fdf81 --- /dev/null +++ b/tests/functional/pagila-schema.sql @@ -0,0 +1,1748 @@ +-- +-- PostgreSQL database dump +-- + +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = off; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET escape_string_warning = off; + +-- +-- Name: plpgsql; Type: PROCEDURAL LANGUAGE; Schema: -; Owner: postgres +-- + +-- CREATE PROCEDURAL LANGUAGE plpgsql; + + +-- ALTER PROCEDURAL LANGUAGE plpgsql OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres; + +SET search_path = public, pg_catalog; + +-- +-- Name: actor_actor_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE actor_actor_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.actor_actor_id_seq OWNER TO postgres; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: actor; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE actor ( + actor_id integer DEFAULT nextval('actor_actor_id_seq'::regclass) NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.actor OWNER TO postgres; + +-- +-- Name: mpaa_rating; Type: TYPE; Schema: public; Owner: postgres +-- + +CREATE TYPE mpaa_rating AS ENUM ( + 'G', + 'PG', + 'PG-13', + 'R', + 'NC-17' +); + + +ALTER TYPE public.mpaa_rating OWNER TO postgres; + +-- +-- Name: year; Type: DOMAIN; Schema: public; Owner: postgres +-- + +CREATE DOMAIN year AS integer + CONSTRAINT year_check CHECK (((VALUE >= 1901) AND (VALUE <= 2155))); + + +ALTER DOMAIN public.year OWNER TO postgres; + +-- +-- Name: _group_concat(text, text); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION _group_concat(text, text) RETURNS text + AS $_$ +SELECT CASE + WHEN $2 IS NULL THEN $1 + WHEN $1 IS NULL THEN $2 + ELSE $1 || ', ' || $2 +END +$_$ + LANGUAGE sql IMMUTABLE; + + +ALTER FUNCTION public._group_concat(text, text) OWNER TO postgres; + +-- +-- Name: group_concat(text); Type: AGGREGATE; Schema: public; Owner: postgres +-- + +CREATE AGGREGATE group_concat(text) ( + SFUNC = _group_concat, + STYPE = text +); + + +ALTER AGGREGATE public.group_concat(text) OWNER TO postgres; + +-- +-- Name: category_category_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE category_category_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.category_category_id_seq OWNER TO postgres; + +-- +-- Name: category; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE category ( + category_id integer DEFAULT nextval('category_category_id_seq'::regclass) NOT NULL, + name character varying(25) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.category OWNER TO postgres; + +-- +-- Name: film_film_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE film_film_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.film_film_id_seq OWNER TO postgres; + +-- +-- Name: film; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film ( + film_id integer DEFAULT nextval('film_film_id_seq'::regclass) NOT NULL, + title character varying(255) NOT NULL, + description text, + release_year year, + language_id smallint NOT NULL, + original_language_id smallint, + rental_duration smallint DEFAULT 3 NOT NULL, + rental_rate numeric(4,2) DEFAULT 4.99 NOT NULL, + length smallint, + replacement_cost numeric(5,2) DEFAULT 19.99 NOT NULL, + rating mpaa_rating DEFAULT 'G'::mpaa_rating, + last_update timestamp without time zone DEFAULT now() NOT NULL, + special_features text[], + fulltext tsvector NOT NULL +); + + +ALTER TABLE public.film OWNER TO postgres; + +-- +-- Name: film_actor; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film_actor ( + actor_id smallint NOT NULL, + film_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.film_actor OWNER TO postgres; + +-- +-- Name: film_category; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film_category ( + film_id smallint NOT NULL, + category_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.film_category OWNER TO postgres; + +-- +-- Name: actor_info; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW actor_info AS + SELECT a.actor_id, a.first_name, a.last_name, group_concat(DISTINCT (((c.name)::text || ': '::text) || (SELECT group_concat((f.title)::text) AS group_concat FROM ((film f JOIN film_category fc ON ((f.film_id = fc.film_id))) JOIN film_actor fa ON ((f.film_id = fa.film_id))) WHERE ((fc.category_id = c.category_id) AND (fa.actor_id = a.actor_id)) GROUP BY fa.actor_id))) AS film_info FROM (((actor a LEFT JOIN film_actor fa ON ((a.actor_id = fa.actor_id))) LEFT JOIN film_category fc ON ((fa.film_id = fc.film_id))) LEFT JOIN category c ON ((fc.category_id = c.category_id))) GROUP BY a.actor_id, a.first_name, a.last_name; + + +ALTER TABLE public.actor_info OWNER TO postgres; + +-- +-- Name: address_address_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE address_address_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.address_address_id_seq OWNER TO postgres; + +-- +-- Name: address; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE address ( + address_id integer DEFAULT nextval('address_address_id_seq'::regclass) NOT NULL, + address character varying(50) NOT NULL, + address2 character varying(50), + district character varying(20) NOT NULL, + city_id smallint NOT NULL, + postal_code character varying(10), + phone character varying(20) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.address OWNER TO postgres; + +-- +-- Name: city_city_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE city_city_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.city_city_id_seq OWNER TO postgres; + +-- +-- Name: city; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE city ( + city_id integer DEFAULT nextval('city_city_id_seq'::regclass) NOT NULL, + city character varying(50) NOT NULL, + country_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.city OWNER TO postgres; + +-- +-- Name: country_country_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE country_country_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.country_country_id_seq OWNER TO postgres; + +-- +-- Name: country; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE country ( + country_id integer DEFAULT nextval('country_country_id_seq'::regclass) NOT NULL, + country character varying(50) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.country OWNER TO postgres; + +-- +-- Name: customer_customer_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE customer_customer_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.customer_customer_id_seq OWNER TO postgres; + +-- +-- Name: customer; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE customer ( + customer_id integer DEFAULT nextval('customer_customer_id_seq'::regclass) NOT NULL, + store_id smallint NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + email character varying(50), + address_id smallint NOT NULL, + activebool boolean DEFAULT true NOT NULL, + create_date date DEFAULT ('now'::text)::date NOT NULL, + last_update timestamp without time zone DEFAULT now(), + active integer +); + + +ALTER TABLE public.customer OWNER TO postgres; + +-- +-- Name: customer_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW customer_list AS + SELECT cu.customer_id AS id, (((cu.first_name)::text || ' '::text) || (cu.last_name)::text) AS name, a.address, a.postal_code AS "zip code", a.phone, city.city, country.country, CASE WHEN cu.activebool THEN 'active'::text ELSE ''::text END AS notes, cu.store_id AS sid FROM (((customer cu JOIN address a ON ((cu.address_id = a.address_id))) JOIN city ON ((a.city_id = city.city_id))) JOIN country ON ((city.country_id = country.country_id))); + + +ALTER TABLE public.customer_list OWNER TO postgres; + +-- +-- Name: film_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW film_list AS + SELECT film.film_id AS fid, film.title, film.description, category.name AS category, film.rental_rate AS price, film.length, film.rating, group_concat((((actor.first_name)::text || ' '::text) || (actor.last_name)::text)) AS actors FROM ((((category LEFT JOIN film_category ON ((category.category_id = film_category.category_id))) LEFT JOIN film ON ((film_category.film_id = film.film_id))) JOIN film_actor ON ((film.film_id = film_actor.film_id))) JOIN actor ON ((film_actor.actor_id = actor.actor_id))) GROUP BY film.film_id, film.title, film.description, category.name, film.rental_rate, film.length, film.rating; + + +ALTER TABLE public.film_list OWNER TO postgres; + +-- +-- Name: inventory_inventory_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE inventory_inventory_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.inventory_inventory_id_seq OWNER TO postgres; + +-- +-- Name: inventory; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE inventory ( + inventory_id integer DEFAULT nextval('inventory_inventory_id_seq'::regclass) NOT NULL, + film_id smallint NOT NULL, + store_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.inventory OWNER TO postgres; + +-- +-- Name: language_language_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE language_language_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.language_language_id_seq OWNER TO postgres; + +-- +-- Name: language; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE language ( + language_id integer DEFAULT nextval('language_language_id_seq'::regclass) NOT NULL, + name character(20) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.language OWNER TO postgres; + +-- +-- Name: nicer_but_slower_film_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW nicer_but_slower_film_list AS + SELECT film.film_id AS fid, film.title, film.description, category.name AS category, film.rental_rate AS price, film.length, film.rating, group_concat((((upper("substring"((actor.first_name)::text, 1, 1)) || lower("substring"((actor.first_name)::text, 2))) || upper("substring"((actor.last_name)::text, 1, 1))) || lower("substring"((actor.last_name)::text, 2)))) AS actors FROM ((((category LEFT JOIN film_category ON ((category.category_id = film_category.category_id))) LEFT JOIN film ON ((film_category.film_id = film.film_id))) JOIN film_actor ON ((film.film_id = film_actor.film_id))) JOIN actor ON ((film_actor.actor_id = actor.actor_id))) GROUP BY film.film_id, film.title, film.description, category.name, film.rental_rate, film.length, film.rating; + + +ALTER TABLE public.nicer_but_slower_film_list OWNER TO postgres; + +-- +-- Name: payment_payment_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE payment_payment_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.payment_payment_id_seq OWNER TO postgres; + +-- +-- Name: payment; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment ( + payment_id integer DEFAULT nextval('payment_payment_id_seq'::regclass) NOT NULL, + customer_id smallint NOT NULL, + staff_id smallint NOT NULL, + rental_id integer NOT NULL, + amount numeric(5,2) NOT NULL, + payment_date timestamp without time zone NOT NULL +); + + +ALTER TABLE public.payment OWNER TO postgres; + +-- +-- Name: payment_p2007_01; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_01 (CONSTRAINT payment_p2007_01_payment_date_check CHECK (((payment_date >= '2007-01-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-02-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_01 OWNER TO postgres; + +-- +-- Name: payment_p2007_02; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_02 (CONSTRAINT payment_p2007_02_payment_date_check CHECK (((payment_date >= '2007-02-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-03-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_02 OWNER TO postgres; + +-- +-- Name: payment_p2007_03; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_03 (CONSTRAINT payment_p2007_03_payment_date_check CHECK (((payment_date >= '2007-03-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-04-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_03 OWNER TO postgres; + +-- +-- Name: payment_p2007_04; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_04 (CONSTRAINT payment_p2007_04_payment_date_check CHECK (((payment_date >= '2007-04-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-05-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_04 OWNER TO postgres; + +-- +-- Name: payment_p2007_05; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_05 (CONSTRAINT payment_p2007_05_payment_date_check CHECK (((payment_date >= '2007-05-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-06-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_05 OWNER TO postgres; + +-- +-- Name: payment_p2007_06; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_06 (CONSTRAINT payment_p2007_06_payment_date_check CHECK (((payment_date >= '2007-06-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-07-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_06 OWNER TO postgres; + +-- +-- Name: rental_rental_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE rental_rental_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.rental_rental_id_seq OWNER TO postgres; + +-- +-- Name: rental; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE rental ( + rental_id integer DEFAULT nextval('rental_rental_id_seq'::regclass) NOT NULL, + rental_date timestamp without time zone NOT NULL, + inventory_id integer NOT NULL, + customer_id smallint NOT NULL, + return_date timestamp without time zone, + staff_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.rental OWNER TO postgres; + +-- +-- Name: sales_by_film_category; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW sales_by_film_category AS + SELECT c.name AS category, sum(p.amount) AS total_sales FROM (((((payment p JOIN rental r ON ((p.rental_id = r.rental_id))) JOIN inventory i ON ((r.inventory_id = i.inventory_id))) JOIN film f ON ((i.film_id = f.film_id))) JOIN film_category fc ON ((f.film_id = fc.film_id))) JOIN category c ON ((fc.category_id = c.category_id))) GROUP BY c.name ORDER BY sum(p.amount) DESC; + + +ALTER TABLE public.sales_by_film_category OWNER TO postgres; + +-- +-- Name: staff_staff_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE staff_staff_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.staff_staff_id_seq OWNER TO postgres; + +-- +-- Name: staff; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE staff ( + staff_id integer DEFAULT nextval('staff_staff_id_seq'::regclass) NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + address_id smallint NOT NULL, + email character varying(50), + store_id smallint NOT NULL, + active boolean DEFAULT true NOT NULL, + username character varying(16) NOT NULL, + password character varying(40), + last_update timestamp without time zone DEFAULT now() NOT NULL, + picture bytea +); + + +ALTER TABLE public.staff OWNER TO postgres; + +-- +-- Name: store_store_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE store_store_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.store_store_id_seq OWNER TO postgres; + +-- +-- Name: store; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE store ( + store_id integer DEFAULT nextval('store_store_id_seq'::regclass) NOT NULL, + manager_staff_id smallint NOT NULL, + address_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.store OWNER TO postgres; + +-- +-- Name: sales_by_store; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW sales_by_store AS + SELECT (((c.city)::text || ','::text) || (cy.country)::text) AS store, (((m.first_name)::text || ' '::text) || (m.last_name)::text) AS manager, sum(p.amount) AS total_sales FROM (((((((payment p JOIN rental r ON ((p.rental_id = r.rental_id))) JOIN inventory i ON ((r.inventory_id = i.inventory_id))) JOIN store s ON ((i.store_id = s.store_id))) JOIN address a ON ((s.address_id = a.address_id))) JOIN city c ON ((a.city_id = c.city_id))) JOIN country cy ON ((c.country_id = cy.country_id))) JOIN staff m ON ((s.manager_staff_id = m.staff_id))) GROUP BY cy.country, c.city, s.store_id, m.first_name, m.last_name ORDER BY cy.country, c.city; + + +ALTER TABLE public.sales_by_store OWNER TO postgres; + +-- +-- Name: staff_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW staff_list AS + SELECT s.staff_id AS id, (((s.first_name)::text || ' '::text) || (s.last_name)::text) AS name, a.address, a.postal_code AS "zip code", a.phone, city.city, country.country, s.store_id AS sid FROM (((staff s JOIN address a ON ((s.address_id = a.address_id))) JOIN city ON ((a.city_id = city.city_id))) JOIN country ON ((city.country_id = country.country_id))); + + +ALTER TABLE public.staff_list OWNER TO postgres; + +-- +-- Name: film_in_stock(integer, integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION film_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) RETURNS SETOF integer + AS $_$ + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND inventory_in_stock(inventory_id); +$_$ + LANGUAGE sql; + + +ALTER FUNCTION public.film_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) OWNER TO postgres; + +-- +-- Name: film_not_in_stock(integer, integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION film_not_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) RETURNS SETOF integer + AS $_$ + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND NOT inventory_in_stock(inventory_id); +$_$ + LANGUAGE sql; + + +ALTER FUNCTION public.film_not_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) OWNER TO postgres; + +-- +-- Name: get_customer_balance(integer, timestamp without time zone); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION get_customer_balance(p_customer_id integer, p_effective_date timestamp without time zone) RETURNS numeric + AS $$ + --#OK, WE NEED TO CALCULATE THE CURRENT BALANCE GIVEN A CUSTOMER_ID AND A DATE + --#THAT WE WANT THE BALANCE TO BE EFFECTIVE FOR. THE BALANCE IS: + --# 1) RENTAL FEES FOR ALL PREVIOUS RENTALS + --# 2) ONE DOLLAR FOR EVERY DAY THE PREVIOUS RENTALS ARE OVERDUE + --# 3) IF A FILM IS MORE THAN RENTAL_DURATION * 2 OVERDUE, CHARGE THE REPLACEMENT_COST + --# 4) SUBTRACT ALL PAYMENTS MADE BEFORE THE DATE SPECIFIED +DECLARE + v_rentfees DECIMAL(5,2); --#FEES PAID TO RENT THE VIDEOS INITIALLY + v_overfees INTEGER; --#LATE FEES FOR PRIOR RENTALS + v_payments DECIMAL(5,2); --#SUM OF PAYMENTS MADE PREVIOUSLY +BEGIN + SELECT COALESCE(SUM(film.rental_rate),0) INTO v_rentfees + FROM film, inventory, rental + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + SELECT COALESCE(SUM(IF((rental.return_date - rental.rental_date) > (film.rental_duration * '1 day'::interval), + ((rental.return_date - rental.rental_date) - (film.rental_duration * '1 day'::interval)),0)),0) INTO v_overfees + FROM rental, inventory, film + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + SELECT COALESCE(SUM(payment.amount),0) INTO v_payments + FROM payment + WHERE payment.payment_date <= p_effective_date + AND payment.customer_id = p_customer_id; + + RETURN v_rentfees + v_overfees - v_payments; +END +$$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.get_customer_balance(p_customer_id integer, p_effective_date timestamp without time zone) OWNER TO postgres; + +-- +-- Name: inventory_held_by_customer(integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION inventory_held_by_customer(p_inventory_id integer) RETURNS integer + AS $$ +DECLARE + v_customer_id INTEGER; +BEGIN + + SELECT customer_id INTO v_customer_id + FROM rental + WHERE return_date IS NULL + AND inventory_id = p_inventory_id; + + RETURN v_customer_id; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.inventory_held_by_customer(p_inventory_id integer) OWNER TO postgres; + +-- +-- Name: inventory_in_stock(integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION inventory_in_stock(p_inventory_id integer) RETURNS boolean + AS $$ +DECLARE + v_rentals INTEGER; + v_out INTEGER; +BEGIN + -- AN ITEM IS IN-STOCK IF THERE ARE EITHER NO ROWS IN THE rental TABLE + -- FOR THE ITEM OR ALL ROWS HAVE return_date POPULATED + + SELECT count(*) INTO v_rentals + FROM rental + WHERE inventory_id = p_inventory_id; + + IF v_rentals = 0 THEN + RETURN TRUE; + END IF; + + SELECT COUNT(rental_id) INTO v_out + FROM inventory LEFT JOIN rental USING(inventory_id) + WHERE inventory.inventory_id = p_inventory_id + AND rental.return_date IS NULL; + + IF v_out > 0 THEN + RETURN FALSE; + ELSE + RETURN TRUE; + END IF; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.inventory_in_stock(p_inventory_id integer) OWNER TO postgres; + +-- +-- Name: last_day(timestamp without time zone); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION last_day(timestamp without time zone) RETURNS date + AS $_$ + SELECT CASE + WHEN EXTRACT(MONTH FROM $1) = 12 THEN + (((EXTRACT(YEAR FROM $1) + 1) operator(pg_catalog.||) '-01-01')::date - INTERVAL '1 day')::date + ELSE + ((EXTRACT(YEAR FROM $1) operator(pg_catalog.||) '-' operator(pg_catalog.||) (EXTRACT(MONTH FROM $1) + 1) operator(pg_catalog.||) '-01')::date - INTERVAL '1 day')::date + END +$_$ + LANGUAGE sql IMMUTABLE STRICT; + + +ALTER FUNCTION public.last_day(timestamp without time zone) OWNER TO postgres; + +-- +-- Name: last_updated(); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION last_updated() RETURNS trigger + AS $$ +BEGIN + NEW.last_update = CURRENT_TIMESTAMP; + RETURN NEW; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.last_updated() OWNER TO postgres; + +-- +-- Name: rewards_report(integer, numeric); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION rewards_report(min_monthly_purchases integer, min_dollar_amount_purchased numeric) RETURNS SETOF customer + AS $_$ +DECLARE + last_month_start DATE; + last_month_end DATE; +rr RECORD; +tmpSQL TEXT; +BEGIN + + /* Some sanity checks... */ + IF min_monthly_purchases = 0 THEN + RAISE EXCEPTION 'Minimum monthly purchases parameter must be > 0'; + END IF; + IF min_dollar_amount_purchased = 0.00 THEN + RAISE EXCEPTION 'Minimum monthly dollar amount purchased parameter must be > $0.00'; + END IF; + + last_month_start := CURRENT_DATE - '3 month'::interval; + last_month_start := to_date((extract(YEAR FROM last_month_start) || '-' || extract(MONTH FROM last_month_start) || '-01'),'YYYY-MM-DD'); + last_month_end := LAST_DAY(last_month_start); + + /* + Create a temporary storage area for Customer IDs. + */ + CREATE TEMPORARY TABLE tmpCustomer (customer_id INTEGER NOT NULL PRIMARY KEY); + + /* + Find all customers meeting the monthly purchase requirements + */ + + tmpSQL := 'INSERT INTO tmpCustomer (customer_id) + SELECT p.customer_id + FROM payment AS p + WHERE DATE(p.payment_date) BETWEEN '||quote_literal(last_month_start) ||' AND '|| quote_literal(last_month_end) || ' + GROUP BY customer_id + HAVING SUM(p.amount) > '|| min_dollar_amount_purchased || ' + AND COUNT(customer_id) > ' ||min_monthly_purchases ; + + EXECUTE tmpSQL; + + /* + Output ALL customer information of matching rewardees. + Customize output as needed. + */ + FOR rr IN EXECUTE 'SELECT c.* FROM tmpCustomer AS t INNER JOIN customer AS c ON t.customer_id = c.customer_id' LOOP + RETURN NEXT rr; + END LOOP; + + /* Clean up */ + tmpSQL := 'DROP TABLE tmpCustomer'; + EXECUTE tmpSQL; + +RETURN; +END +$_$ + LANGUAGE plpgsql SECURITY DEFINER; + + +ALTER FUNCTION public.rewards_report(min_monthly_purchases integer, min_dollar_amount_purchased numeric) OWNER TO postgres; + +-- +-- Name: actor_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY actor + ADD CONSTRAINT actor_pkey PRIMARY KEY (actor_id); + + +-- +-- Name: address_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY address + ADD CONSTRAINT address_pkey PRIMARY KEY (address_id); + + +-- +-- Name: category_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY category + ADD CONSTRAINT category_pkey PRIMARY KEY (category_id); + + +-- +-- Name: city_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY city + ADD CONSTRAINT city_pkey PRIMARY KEY (city_id); + + +-- +-- Name: country_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY country + ADD CONSTRAINT country_pkey PRIMARY KEY (country_id); + + +-- +-- Name: customer_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_pkey PRIMARY KEY (customer_id); + + +-- +-- Name: film_actor_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_pkey PRIMARY KEY (actor_id, film_id); + + +-- +-- Name: film_category_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_pkey PRIMARY KEY (film_id, category_id); + + +-- +-- Name: film_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_pkey PRIMARY KEY (film_id); + + +-- +-- Name: inventory_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_pkey PRIMARY KEY (inventory_id); + + +-- +-- Name: language_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY language + ADD CONSTRAINT language_pkey PRIMARY KEY (language_id); + + +-- +-- Name: payment_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_pkey PRIMARY KEY (payment_id); + + +-- +-- Name: rental_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_pkey PRIMARY KEY (rental_id); + + +-- +-- Name: staff_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_pkey PRIMARY KEY (staff_id); + + +-- +-- Name: store_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_pkey PRIMARY KEY (store_id); + + +-- +-- Name: film_fulltext_idx; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX film_fulltext_idx ON film USING gist (fulltext); + + +-- +-- Name: idx_actor_last_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_actor_last_name ON actor USING btree (last_name); + + +-- +-- Name: idx_fk_address_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_address_id ON customer USING btree (address_id); + + +-- +-- Name: idx_fk_city_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_city_id ON address USING btree (city_id); + + +-- +-- Name: idx_fk_country_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_country_id ON city USING btree (country_id); + + +-- +-- Name: idx_fk_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_customer_id ON payment USING btree (customer_id); + + +-- +-- Name: idx_fk_film_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_film_id ON film_actor USING btree (film_id); + + +-- +-- Name: idx_fk_inventory_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_inventory_id ON rental USING btree (inventory_id); + + +-- +-- Name: idx_fk_language_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_language_id ON film USING btree (language_id); + + +-- +-- Name: idx_fk_original_language_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_original_language_id ON film USING btree (original_language_id); + + +-- +-- Name: idx_fk_payment_p2007_01_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_01_customer_id ON payment_p2007_01 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_01_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_01_staff_id ON payment_p2007_01 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_02_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_02_customer_id ON payment_p2007_02 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_02_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_02_staff_id ON payment_p2007_02 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_03_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_03_customer_id ON payment_p2007_03 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_03_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_03_staff_id ON payment_p2007_03 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_04_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_04_customer_id ON payment_p2007_04 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_04_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_04_staff_id ON payment_p2007_04 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_05_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_05_customer_id ON payment_p2007_05 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_05_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_05_staff_id ON payment_p2007_05 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_06_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_06_customer_id ON payment_p2007_06 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_06_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_06_staff_id ON payment_p2007_06 USING btree (staff_id); + + +-- +-- Name: idx_fk_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_staff_id ON payment USING btree (staff_id); + + +-- +-- Name: idx_fk_store_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_store_id ON customer USING btree (store_id); + + +-- +-- Name: idx_last_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_last_name ON customer USING btree (last_name); + + +-- +-- Name: idx_store_id_film_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_store_id_film_id ON inventory USING btree (store_id, film_id); + + +-- +-- Name: idx_title; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_title ON film USING btree (title); + + +-- +-- Name: idx_unq_manager_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX idx_unq_manager_staff_id ON store USING btree (manager_staff_id); + + +-- +-- Name: idx_unq_rental_rental_date_inventory_id_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX idx_unq_rental_rental_date_inventory_id_customer_id ON rental USING btree (rental_date, inventory_id, customer_id); + + +-- +-- Name: payment_p2007_01_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_01_payment_id_unq ON payment_p2007_01 USING btree (payment_id); + + +-- +-- Name: payment_p2007_02_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_02_payment_id_unq ON payment_p2007_02 USING btree (payment_id); + + +-- +-- Name: payment_p2007_03_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_03_payment_id_unq ON payment_p2007_03 USING btree (payment_id); + + +-- +-- Name: payment_p2007_04_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_04_payment_id_unq ON payment_p2007_04 USING btree (payment_id); + + +-- +-- Name: payment_p2007_05_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_05_payment_id_unq ON payment_p2007_05 USING btree (payment_id); + + +-- +-- Name: payment_p2007_06_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_06_payment_id_unq ON payment_p2007_06 USING btree (payment_id); + + +-- +-- Name: payment_insert_p2007_01; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_01 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-01-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-02-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_01 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_02; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_02 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-02-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-03-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_02 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_03; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_03 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-03-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-04-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_03 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_04; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_04 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-04-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-05-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_04 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_05; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_05 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-05-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-06-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_05 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_06; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_06 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-06-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-07-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_06 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: film_fulltext_trigger; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER film_fulltext_trigger + BEFORE INSERT OR UPDATE ON film + FOR EACH ROW + EXECUTE PROCEDURE tsvector_update_trigger('fulltext', 'pg_catalog.english', 'title', 'description'); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON actor + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON address + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON category + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON city + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON country + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON customer + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film_actor + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film_category + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON inventory + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON language + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON rental + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON staff + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON store + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: address_city_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY address + ADD CONSTRAINT address_city_id_fkey FOREIGN KEY (city_id) REFERENCES city(city_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: city_country_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY city + ADD CONSTRAINT city_country_id_fkey FOREIGN KEY (country_id) REFERENCES country(country_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: customer_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: customer_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_actor_actor_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_actor_id_fkey FOREIGN KEY (actor_id) REFERENCES actor(actor_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_actor_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_category_category_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_category_id_fkey FOREIGN KEY (category_id) REFERENCES category(category_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_category_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_language_id_fkey FOREIGN KEY (language_id) REFERENCES language(language_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_original_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_original_language_id_fkey FOREIGN KEY (original_language_id) REFERENCES language(language_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: inventory_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: inventory_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: payment_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: payment_p2007_01_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_01_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_01_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_02_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_02_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_02_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_03_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_03_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_03_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_04_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_04_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_04_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_05_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_05_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_05_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_06_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_06_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_06_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id) ON UPDATE CASCADE ON DELETE SET NULL; + + +-- +-- Name: payment_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_inventory_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_inventory_id_fkey FOREIGN KEY (inventory_id) REFERENCES inventory(inventory_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: staff_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: staff_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id); + + +-- +-- Name: store_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: store_manager_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_manager_staff_id_fkey FOREIGN KEY (manager_staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: public; Type: ACL; Schema: -; Owner: postgres +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/tests/functional/test_autodoc.py b/tests/functional/test_autodoc.py new file mode 100644 index 0000000..374324c --- /dev/null +++ b/tests/functional/test_autodoc.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using autodoc schema + +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/~checkout~/autodoc/autodoc/ +regressdatabase.sql?rev=1.2 +""" +from pyrseas.testutils import DbMigrateTestCase + + +class AutodocTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('autodoc') + cls.remove_tempfiles('empty') + + def test_autodoc(self): + # Create the source schema + self.execute_script(__file__, 'autodoc-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('autodoc-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + emptydump = self.tempfile_path('empty.dump') + self.run_pg_dump(emptydump) + emptyyaml = self.tempfile_path('empty.yaml') + self.create_yaml(emptyyaml) + + # Migrate the target database + targsql = self.tempfile_path('autodoc.sql') + self.migrate_target(srcyaml, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc.yaml') + self.create_yaml(targyaml) + + # diff autodoc-src.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff autodoc-src.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.migrate_target(emptyyaml, targsql) + + # Run pg_dump against target database + self.run_pg_dump(targdump) + + # Create target YAML file + self.create_yaml(targyaml) + + # diff empty.dump against autodoc.dump + assert self.lines(emptydump) == self.lines(targdump) + # diff empty.yaml against autodoc.yaml + assert self.lines(emptyyaml) == self.lines(targyaml) diff --git a/tests/functional/test_autodoc_dir.py b/tests/functional/test_autodoc_dir.py new file mode 100644 index 0000000..92b9d79 --- /dev/null +++ b/tests/functional/test_autodoc_dir.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using autodoc schema but I/O to/from a directory + +Same as test_autodoc.py but with directory tree instead of a single YAML file. +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/~checkout~/autodoc/autodoc/ +regressdatabase.sql?rev=1.2 +""" +from pyrseas.testutils import DbMigrateTestCase + + +class AutodocTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + self.remove_tempfiles('metadata') + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('autodoc') + cls.remove_tempfiles('metadata') + + def test_autodoc(self): + # Create the source schema + self.execute_script(__file__, 'autodoc-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file and directory tree + # Note: the single YAML file is for verification against the target, + # the YAML directory tree is for processing + srcyaml = self.tempfile_path('autodoc-src.yaml') + self.create_yaml(srcyaml, True) + self.create_yaml(None, True) + + # Migrate the target database + targsql = self.tempfile_path('autodoc.sql') + self.migrate_target(None, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc.yaml') + self.create_yaml(targyaml) + + # diff autodoc-src.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff autodoc-src.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.srcdb.execute_commit( + "DROP SCHEMA inherit, product, store, warehouse CASCADE") + # Create source YAML file and directory tree + srcyaml = self.tempfile_path('autodoc-src-empty.yaml') + self.create_yaml(srcyaml, True) + self.create_yaml(None, True) + targsql = self.tempfile_path('autodoc-empty.sql') + self.migrate_target(None, targsql) + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src-empty.dump') + self.run_pg_dump(srcdump, True) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc-empty.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc-empty.yaml') + self.create_yaml(targyaml) + + # diff empty.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff empty.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) diff --git a/tests/functional/test_filmversions.py b/tests/functional/test_filmversions.py new file mode 100644 index 0000000..d51a55f --- /dev/null +++ b/tests/functional/test_filmversions.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using film versions schema + +See https://pyrseas.wordpress.com/2011/02/07/ +version-control-part-2-sql-databases/ +""" +import os + +from pyrseas.testutils import DbMigrateTestCase +from pyrseas.yamlutil import yamldump + + +class FilmTestCase(DbMigrateTestCase): + + def setUp(self): + self.remove_public_schema(self.srcdb) + + @classmethod + def tearDownClass(cls): + cls.remove_tempfiles('film-0.') + cls.remove_tempfiles('usercfg.yaml') + cls.remove_tempfiles('config.yaml') + cls.remove_tempfiles('metadata') + + def test_film_version_01(self): + "Create schema version 0.1" + self.execute_script(__file__, 'film-schema-0.1.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.1-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('film-0.1-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + self.run_pg_dump(self.tempfile_path('film-0.0.dump')) + self.create_yaml(self.tempfile_path('film-0.0.yaml')) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.1.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.1.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.1.yaml') + self.create_yaml(targyaml) + + # diff film-0.1-src.dump against film-0.1.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.1-src.yaml against film-0.1.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_02(self): + "Update schema to version 0.2" + self.execute_script(__file__, 'film-schema-0.2.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.2-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('film-0.2-src.yaml') + self.create_yaml(srcyaml, True) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.2.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.2.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.2.yaml') + self.create_yaml(targyaml) + + # diff film-0.2-src.dump against film-0.2.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.2-src.yaml against film-0.2.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_03(self): + "Update schema to version 0.3" + self.execute_script(__file__, 'film-schema-0.3a.sql') + self.execute_script(__file__, 'film-schema-0.3b.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.3-src.dump') + self.run_pg_dump(srcdump, True, True) + + # Create source YAML file + usercfg = self.tempfile_path("usercfg.yaml") + with open(usercfg, 'w') as f: + f.write(yamldump({'repository': {'path': self.tempfile_path('')}})) + os.environ["PYRSEAS_USER_CONFIG"] = usercfg + with open(self.tempfile_path("config.yaml"), 'w') as f: + f.write(yamldump({'datacopy': {'schema sd': ['genre']}})) + srcyaml = self.tempfile_path('film-0.3-src.yaml') + self.create_yaml(srcyaml, True) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.3.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.3.dump') + self.run_pg_dump(targdump, False, True) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.3.yaml') + self.create_yaml(targyaml) + + # diff film-0.3-src.dump against film-0.3.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.3-src.yaml against film-0.3.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_04(self): + "Revert to schema version 0.2" + srcyaml = self.tempfile_path('film-0.2.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.3-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.3-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.3-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.2.dump against film-0.3-undo.dump + srcdump = self.tempfile_path('film-0.2.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.2.yaml against film-0.3-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_05(self): + "Revert to schema version 0.1" + srcyaml = self.tempfile_path('film-0.1.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.2-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.2-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.2-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.1.dump against film-0.2-undo.dump + srcdump = self.tempfile_path('film-0.1.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.1.yaml against film-0.2-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_06(self): + "Revert to empty schema" + srcyaml = self.tempfile_path('film-0.0.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.1-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.1-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.1-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.0.dump against film-0.1-undo.dump + srcdump = self.tempfile_path('film-0.0.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.0.yaml against film-0.1-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) diff --git a/tests/functional/test_pagila.py b/tests/functional/test_pagila.py new file mode 100644 index 0000000..984269b --- /dev/null +++ b/tests/functional/test_pagila.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using pagila schema + +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/dbsamples/pagila/ +pagila-schema.sql?rev=1.8 +""" +from difflib import unified_diff + +from pyrseas.testutils import DbMigrateTestCase + + +class PagilaTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('pagila') + cls.remove_tempfiles('empty') + + def test_pagila(self): + # Create the source schema + self.execute_script(__file__, 'pagila-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('pagila-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('pagila-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + emptydump = self.tempfile_path('empty.dump') + self.run_pg_dump(emptydump) + emptyyaml = self.tempfile_path('empty.yaml') + self.create_yaml(emptyyaml) + + # Migrate the target database + targsql = self.tempfile_path('pagila.sql') + self.migrate_target(srcyaml, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('pagila.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('pagila.yaml') + self.create_yaml(targyaml) + + # diff pagila-src.dump against pagila.dump + # order of triggers requires special handling + adds = [] + subs = [] + for line in unified_diff(self.lines(srcdump), self.lines(targdump)): + if line == '--- \n' or line == '+++ \n' or line.startswith('@@'): + continue + if line[:1] == '+': + adds.append(line[1:-1]) + elif line[:1] == '-': + subs.append(line[1:-1]) + subs = sorted(subs) + for i, line in enumerate(sorted(adds)): + assert line == subs[i] + # diff pagila-src.yaml against pagila.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.migrate_target(emptyyaml, targsql) + + # Workaround problem with privileges on schema public + self.db.execute("GRANT ALL ON SCHEMA public TO postgres") + self.db.conn.commit() + # Run pg_dump against target database + self.run_pg_dump(targdump) + + self.db.execute("REVOKE ALL ON SCHEMA public FROM postgres") + self.db.conn.commit() + # Create target YAML file + self.create_yaml(targyaml) + + # diff empty.dump against pagila.dump + assert self.lines(emptydump) == self.lines(targdump) + # diff empty.yaml against pagila.yaml + assert self.lines(emptyyaml) == self.lines(targyaml) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..faab758 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""Test configuration files""" + +import os +import sys + +from pyrseas.config import Config +from pyrseas.cmdargs import cmd_parser, parse_args +from pyrseas.yamlutil import yamldump + +USER_CFG_DATA = {'database': {'port': 5433}, + 'output': {'version_comment': True}} +CFG_TABLE_DATA = {'schema public': ['t1', 't2']} +CFG_DATA = {'datacopy': CFG_TABLE_DATA} +CFG_FILE = 'testcfg.yaml' + + +def test_defaults(): + "Create a configuration with defaults" + cfg = Config() + for key in ['audit_columns', 'functions', 'function_templates', 'columns', + 'triggers']: + assert key in cfg['augmenter'] + for key in ['metadata', 'data']: + assert key in cfg['repository'] + + +def test_user_config(tmpdir): + "Test a user configuration file" + f = tmpdir.join(CFG_FILE) + f.write(yamldump(USER_CFG_DATA)) + os.environ["PYRSEAS_USER_CONFIG"] = f.strpath + cfg = Config() + assert cfg['database'] == {'port': 5433} + assert cfg['output'] == {'version_comment': True} + + +def test_repo_config(tmpdir): + "Test a repository configuration file" + ucfg = tmpdir.join(CFG_FILE) + ucfg.write(yamldump({'repository': {'path': tmpdir.strpath}})) + f = tmpdir.join("config.yaml") + f.write(yamldump(CFG_DATA)) + os.environ["PYRSEAS_USER_CONFIG"] = ucfg.strpath + cfg = Config() + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_cmd_parser(tmpdir): + "Test parsing a configuration file specified on the command line" + f = tmpdir.join(CFG_FILE) + f.write(yamldump(CFG_DATA)) + sys.argv = ['testprog', 'testdb', '--config', f.strpath] + os.environ["PYRSEAS_USER_CONFIG"] = '' + parser = cmd_parser("Test description", '0.0.1') + cfg = parse_args(parser) + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_parse_repo_config(tmpdir): + "Test parsing a repository configuration file in the current directory" + f = tmpdir.join('config.yaml') + f.write(yamldump(CFG_DATA)) + os.chdir(tmpdir.strpath) + sys.argv = ['testprog', 'testdb'] + os.environ["PYRSEAS_USER_CONFIG"] = '' + parser = cmd_parser("Test description", '0.0.1') + cfg = parse_args(parser) + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_repo_user_config(tmpdir): + "Test a repository path specified in the user config" + usercfg = {'repository': {'path': tmpdir.strpath}} + userf = tmpdir.join("usercfg.yaml") + userf.write(yamldump(usercfg)) + os.environ["PYRSEAS_USER_CONFIG"] = userf.strpath + repof = tmpdir.join("config.yaml") + repof.write(yamldump(CFG_DATA)) + cfg = Config() + assert cfg['datacopy'] == CFG_TABLE_DATA diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..bc91045 --- /dev/null +++ b/tox.ini @@ -0,0 +1,34 @@ +[pg150] +setenv = + PYRSEAS_TEST_PORT={env:PG150_PORT} + +[pg130] +setenv = + PYRSEAS_TEST_PORT={env:PG130_PORT} + +[pg110] +setenv = + PYRSEAS_TEST_PORT={env:PG110_PORT} + +[testenv:py39pg150] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg150]setenv} + +[testenv:py39pg130] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg130]setenv} + +[testenv:py39pg110] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg110]setenv} + +[testenv] +deps=pytest +setenv = + PYTHONPATH = {toxinidir} +passenv = HOME +commands = + py.test tests