commit 366d7865ba5e554e35417ca0947b965bf87bd0d4 Author: Wendell Jones Date: Tue May 20 09:24:29 2025 -0400 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ed7ff9f --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.pyc +*~ +MANIFEST +build +dist +docs/_build +Pyrseas.egg-info +.coverage +htmlcov +.tox +.cache +/*.egg +venv diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..e336df8 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: requirements.txt + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5f2fe43 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,50 @@ +language: python + +addons: + apt: + update: true + packages: postgresql-common + sources: + - sourceline: deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main + +jobs: + include: + - python: "2.7" + env: POSTGRES=9.4 + - python: "2.7" + env: POSTGRES=9.5 + - python: "2.7" + env: POSTGRES=9.6 + - python: "3.6" + env: POSTGRES=10 + - python: "3.7" + env: POSTGRES=11 + - python: "3.8" + env: POSTGRES=12 + +# Ensure the desired version of Postgres is installed and running +# Most of this is normally handled by Travis automatically, but only for certain versions of Postgres +# https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash +before_install: + - sudo systemctl stop postgresql + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install postgresql-$POSTGRES postgresql-plperl-$POSTGRES postgresql-plpython3-$POSTGRES + # the port may have been auto-configured to use 5433 if it thought 5422 was already in use + - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf + # postgresql-11+ default to "peer" + - sudo sed -i -E -e 's/^local\s+all\s+postgres\s+peer/local all postgres trust/' /etc/postgresql/*/main/pg_hba.conf + - sudo mkdir -p /var/ramfs/postgresql + - if [ ! -d /var/ramfs/postgresql/$POSTGRES ]; then sudo cp -rp /var/lib/postgresql/$POSTGRES /var/ramfs/postgresql/$POSTGRES; fi + - sudo systemctl start postgresql@$POSTGRES-main + - (cd /; sudo -u postgres createuser -e -s travis || echo failed createuser) + - (cd /; sudo -u postgres createdb -e -O travis travis || echo failed created) + +# Ensure Postgres is configured as described in docs/testing.rst +before_script: + - sudo locale-gen --no-archive fr_FR.UTF-8 + - sudo mkdir -p /extra/pg/ts1 /extra/pg/ts2 + - sudo chown postgres:postgres /extra/pg/ts1 /extra/pg/ts2 + - psql -Upostgres -c "CREATE TABLESPACE ts1 LOCATION '/extra/pg/ts1'" + - psql -Upostgres -c "CREATE TABLESPACE ts2 LOCATION '/extra/pg/ts2'" + +script: + - python setup.py test diff --git a/AUTHORS.rst b/AUTHORS.rst new file mode 100644 index 0000000..1b48fb8 --- /dev/null +++ b/AUTHORS.rst @@ -0,0 +1,29 @@ +Pyrseas was started in 2010. + +The PRIMARY AUTHORS are or have been: + + * Joe Abbate + * Ronan Dunklau + * Douglas Fraser + * Mark Goldfinch + * Ryan Graham + * Andjelko Horvat + * Roger Hunwicks + * Ghislain Leveque + * Chris Mayo + * Philippe Pepiot + * Andrey Popp + * Ross J. Reedstrom + * github.com/shirkey + * Denis Smirnov + * Feike Steenbergen + * Paulo Tioseco + * Daniele Varrazzo + * Vasiliy Yeremeyev + +A big THANK YOU goes to the following individuals that provided +inspiration for some of the features and design of Pyrseas: + + * Ken Downs for creating the Andromeda project. + + * Robert Brewer for Post-Facto. diff --git a/Changelog.rst b/Changelog.rst new file mode 100644 index 0000000..3f45a81 --- /dev/null +++ b/Changelog.rst @@ -0,0 +1,347 @@ +0.10.0 (7-Nov-2022) + +Convert Pyrseas to use Psycopg version 3. Remove support for Python +2.x and Postgres versions prior to PG 10. + +Reinstate code for interfacing with psycopg, removing the dependency +on pgdbconn. + +Fixed various issues, including: + + * Partial fix for database objects owned by extensions. (#236) + + * Deal with pg_pltemplate catalog being removed in PG 13. (#226, #227) + + * Problem with trigger procedures with arguments. (#219) + +0.9.1 (14-Apr-2020) + +Expanded the number of keywords that are quoted if they are present in +generated SQL statements. (#212) + +Various changes to support Postgres 12. + +Fixed various issues, including: + + * Handling of multiple foreign key constraints (#210) + + * Column qualifications in Extensions query (#208) + +0.9.0 (22-Jul-2019) + +Schema 'public' is no longer treated as a special case (176) + +Yamltodb was changed to support Postgres 11, in particular due +to changes to the 'pg_proc' catalog (#195) + +Dbtoyaml now outputs column information for views, both regular and +materialized (184) + +Fixed various issues, including: + + * Do not assume constraints always refer to columns (#188) + + * Add non-'public' schemas to search_path to avoid problems with + some extensions (e.g., PostGIS) that need them to resolve + objects (#91) + + * Prevent a view definition from changing a column type (#90) + + * Inconsistent handling of FOREIGN KEY and UNIQUE constraints + leading to AttributeErrors (#182) + + * Recreation of tsvector triggers when columns are added (#179) + +0.8.0 (12-Dec-2017) + +Significant rearchitecture of methods to generate SQL. + + * An object dependency graph is built and traversed to generate SQL + in correct order (#72, #86, #100) + +Added support for Postgres 10, specifically: + + * Table partitioning syntax (#163) + + * Column specification GENERATED AS IDENTITY (#164) + +Added support for other Postgres features: + + * Parallel safe functions and partial aggregation (#161) + + * RANGE types (#173) + + * ALTER TYPE ADD VALUE for changes to ENUM types (#87) + + +0.7.2 (23-Jan-2015) + + Fixed various issues, including: + + * Do not error on tables whose names start with 'public' (#109) + + * Deal properly with inherited constraints in children tables (#102) + + * Handle external languages like plv8 correctly (#97) + + * Correct quoting of mixed case constraint names (#83) + + * Avoid problems with certain complex index definitions (#98) + + * Have dbtoyaml output correctly a table with an embedded period in + the name and having an associated sequence (#79) + + * Use relative paths in database summary for ``--multiple-files`` + (#93) + + * Support mapping of indexes on materialized views (#82) + +0.7.1 (5-Dec-2013) + + * Moved ``config.yaml`` under ``pyrseas`` directory and use + ``package_data`` to install (#77) + + * yamltodb output to a file is encoded using utf-8 (#78) + + +0.7.0 (25-Nov-2013) + + * Added support for: + + - Postgres 9.3, specifically + + + EVENT TRIGGER + + MATERIALIZED VIEWS + + - CLUSTER + - Partial indexes + - Storage parameters in CREATE and ALTER TABLE + - ALTER COLUMN SET STATISTICS + - LEAKPROOF qualifier for FUNCTIONs + - YAML multi-line string formatting for view definitions, + function source text and object comments + + * Configuration files + + All Pyrseas utilities can now use YAML-formatted configuration + files, in addition to command line options + + * Multiple-file input or output + + Spread database object information across a version control + repository + + * Data export/import + + Load a database with static data in production or data subsets + for testing + + * dbtoyaml/yamltodb + + - Added --quote-reserved option to yamltodb + - Exclude arguments from sfunc and finalfunc attributes of + aggregate functions (#54) + - Correct generation of SQL for functions with DEFAULT + arguments (#52) + + * Augmenter + + New utility (dbaugment) to consistently add objects to an + existent database. This is currently an experimental + feature and covers adding audit columns to tables. + + * TTM-inspired relational interface + + A new interface to Postgres, inspired by *The Third Manifesto* + + +0.6.1 (31-Jan-2013) + + * Add support for INSTEAD OF triggers on views (#50). + + * Eliminated yamltodb generation of spurious REVOKE/GRANT commands + (#51). + + * Removed setuptools from setup.py install_requires. + + +0.6.0 (26-Oct-2012) + + * Added support for: + + - EXTENSIONs + - COLLATIONs + - OWNER information + - Access privileges (GRANT and REVOKE) + - TABLESPACEs for tables, primary keys and indexes + - MATCH attributes for foreign keys (#34) + - ALTER composite TYPE ADD/DROP/RENAME ATTRIBUTE + - ENUMs with no labels (#31) + - UNLOGGED tables (#45) + - CREATE FUNCTION SET configuration_parameter (#46) + - PostgreSQL 9.2 + + * Correctly support index functions/expressions (#3, #44). + + * Schema-qualify composite types when dropping or renaming + attributes (#47) + + * Fix DbConnection exception handling under Python 3 (#25). + + * dbtoyaml + + - Fix -t option to output sequences owned by table and the schema + description. + - Use pg_user_mappings view to allow usage by non-superusers. + + * yamltodb + + - Schema-qualify table when dropping columns (#26). + - Correct column drop/add case in middle of table (#8). + - Fix adding and dropping of columns in inherited tables (#33). + - Enable renaming of indexes (#38). + - Ignore all temp schemas (#37) + + * dbtoyaml/yamltodb + + - Give PGUSER precedence over USER environment variable. + + * Testing + + - Added support, via Tox, for testing against multiple + PostgreSQL/Python combinations + + - Changes and documentation for testing on Microsoft Windows + + +0.5.0 (10-Mar-2012) + + * Added support for: + + - TEXTSEARCH parsers, dictionaries, configurations and templates + - FOREIGN DATA WRAPPERs, SERVERs, USER MAPPINGs and FOREIGN TABLEs + - ROWS clause in set-returning functions (issue #11) + - Deferrable/deferred constraints (#13) + - CATEGORY and PREFERRED clauses for TYPEs, + SORTOP clause for AGGREGATEs + HASHES and MERGES clauses for OPERATORs (#15) + - Operator class qualifiers for INDEXes (#16) + - Python 3.2 and later + + * Correct schema normalization for constraints (#9) and indexes. + + * Fix COMMENTs generated for constraints (#12). + + * Fix DEFAULT clause for OPERATOR CLASS. + + * dbtoyaml + + - When restricting to specific schemas or tables, include + non-schema objects (e.g., languages). + + * yamltodb + + - Add -n/--schema option (#6). + - Add -u/--update option to apply SQL statements to target + database. + - Exclude database-wide objects when -n/--schema is used (#21). + - Allow YAML spec argument to be read from standard input. + + * dbtoyaml/yamltodb + + - Add -o/--output option + - Add -W/--password option (#18) + + +0.4.1 (27-Oct-2011) + + * Make the initial SET search_path persistent. + + * Correct exclusion of PG internal schemas in various queries. + + * Fix generation of COMMENTs with single quotes in the text. + + * For inherited tables, only generate constraints that are defined + locally. + + * Correct generation of ALTER TABLE ADD/DROP COLUMN when input + columns are in different order than original. + + * Support PG 9.1 (add description for PL/pgSQL language). + + +0.4.0 (26-Sep-2011) + + * Added support for: + + - CASTs + - CONSTRAINT TRIGGERs + - CONVERSIONs + - OPERATORs, OPERATOR CLASSes and OPERATOR FAMILies + - Dynamically loaded C language functions + - Composite and base TYPEs + + * Clean up and enhance documentation and redundant methods. + + * Use obj_description/col_description functions instead of querying + pg_description directly. + + +0.3.1 (26-Aug-2011) + + * Added workaround for incorrect assumption that 'public' schema is + always present (issue #4). + + * Added support for delimited (or quoted) identifiers, e.g., those + with embedded spaces, upper case characters, etc. (except for SQL + keywords) (issue #5). + + +0.3.0 (30-Jun-2011) + + * Added support for: + + - AGGREGATE functions + - DOMAINs + - ENUMerated TYPEs + - Functions returning table row types + - INDEXes on expressions (issue #3) + - Rewrite RULEs + - SECURITY DEFINER functions + - TRIGGERs + + * Enhanced host/port defaults to use sockets, resulting in noticeable + performance improvement. + + +0.2.1 (7-Jun-2011) + + * Fixed problem with mapping a FOREIGN KEY in a table with a dropped + column (issue #2). + + +0.2.0 (19-May-2011) + + * Added support for: + + - COMMENTs on schemas, tables, columns and functions + - FOREIGN KEY ON UPDATE and ON DELETE actions + - ALTER TABLE RENAME COLUMN and enhanced support for other ALTER + object RENAME statements. + - VIEWs + - INHERITed tables, and by extension, partitioned tables. + - PROCEDURAL LANGUAGEs + - FUNCTIONs. + + * Added files for release via PGXN. + * Added support for testing against multiple PostgreSQL versions. + * Fixed cross-schema REFERENCES failure in dbtoyaml (issue #1). + + +0.1.0 (5-Apr-2011) + + * Initial release + + - dbtoyaml and yamltodb support PostgreSQL schemas, tables, + sequences, check constraints, primary keys, foreign keys, unique + constraints and indexes. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4a47a19 --- /dev/null +++ b/LICENSE @@ -0,0 +1,29 @@ +Copyright (c) 2010 by Joe Abbate, see AUTHORS for more details. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the Pyrseas project nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f68f227 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include README.rst AUTHORS.rst Changelog.rst LICENSE Makefile META.json +recursive-include tests *.py +recursive-include docs * +prune docs/_build diff --git a/META.json b/META.json new file mode 100644 index 0000000..22470e1 --- /dev/null +++ b/META.json @@ -0,0 +1,57 @@ +{ + "name": "Pyrseas", + "abstract": "Utilities to assist in database schema versioning", + "description": "Pyrseas provides utilities to describe a PostgreSQL database schema as YAML, to verify the schema against the same or a different database and to generate SQL that will modify the schema to match the YAML description. Supports PostgreSQL version 10 through 15.", + "version": "0.10.0", + "maintainer": "Joe Abbate ", + "license": "bsd", + "prereqs": { + "runtime": { + "requires": { + "PostgreSQL": "9.4.0" + } + } + }, + "provides": { + "augmentdb": { + "file": "pyrseas/augmentdb.py", + "docfile": "docs/augmentdb.rst", + "version": "0.10.0", + "abstract": "Generates an augmented YAML description of a Postgres database from its catalogs and an augmentation specification" + }, + "dbtoyaml": { + "file": "pyrseas/dbtoyaml.py", + "docfile": "docs/dbtoyaml.rst", + "version": "0.10.0", + "abstract": "Outputs a YAML description of a Postgres database's tables and other objects (metadata), suitable for storing in a version control repository" + }, + "yamltodb": { + "file": "pyrseas/yamltodb.py", + "docfile": "docs/yamltodb.rst", + "version": "0.10.0", + "abstract": "Generates SQL statements to modify a database so that it will match an input YAML/JSON specification" + } + }, + "resources": { + "homepage": "https://perseas.github.io/", + "bugtracker": { + "web": "https://github.com/perseas/Pyrseas/issues" + }, + "repository": { + "url": "git://github.com/perseas/Pyrseas.git", + "web": "https://github.com/perseas/Pyrseas", + "type": "git" + } + }, + "generated_by": "Joe Abbate", + "meta-spec": { + "version": "1.0.0", + "url": "http://pgxn.org/meta/spec.txt" + }, + "tags": [ + "version control", + "yaml", + "database version control", + "schema versioning" + ] +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8700b09 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +# +# Pyrseas Makefile +# + +PYTHON = python3 + +.PHONY: all build docs install installcheck check clean + +all: + $(PYTHON) setup.py build + +build: + $(PYTHON) setup.py sdist --format=gztar,zip + $(PYTHON) setup.py bdist_wheel + +docs: + $(MAKE) -C docs html + +install: + $(PYTHON) setup.py install + +installcheck check: + $(PYTHON) setup.py test + +clean: + $(MAKE) -C docs clean + $(PYTHON) setup.py clean diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1857d8b --- /dev/null +++ b/README.rst @@ -0,0 +1,43 @@ +======= +Pyrseas +======= + +.. image:: https://api.travis-ci.org/perseas/Pyrseas.png?branch=master + :target: https://travis-ci.com/perseas/Pyrseas + +Pyrseas provides utilities to describe a PostgreSQL database schema as +YAML, to verify the schema against the same or a different database +and to generate SQL that will modify the schema to match the YAML +description. + +Features +-------- + +- Outputs a YAML description of a Postgres database's tables + and other objects (metadata), suitable for storing in a version + control repository + +- Generates SQL statements to modify a database so that it will match + an input YAML/JSON specification + +- Generates an augmented YAML description of a Postgres database + from its catalogs and an augmentation specification. + +Requirements +------------ + +- PostgreSQL 10 or higher + +- Python 3.7 or higher + +License +------- + +Pyrseas is free (libre) software and is distributed under the BSD +license. Please see the LICENSE file for details. + +Documentation +------------- + +Please visit `Read the Docs `_ +for the latest documentation. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..eef3aaa --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,89 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Pyrseas.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Pyrseas.qhc" + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ + "run these through (pdf)latex." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/augmentdb.rst b/docs/augmentdb.rst new file mode 100644 index 0000000..16dcd13 --- /dev/null +++ b/docs/augmentdb.rst @@ -0,0 +1,30 @@ +Augmenter Databases +=================== + +.. module:: pyrseas.augmentdb + +The :mod:`augmentdb` module defines the class :class:`AugmentDatabase`. + +Augmenter Database +------------------ + +An :class:`AugmentDatabase` is derived from +:class:`~pyrseas.database.Database`. It contains two "dictionary" +objects. + +One is the :class:`Dicts` container from its parent class. The `db` +Dicts object, defines the database schemas, including their tables and +other objects, by querying the system catalogs. + +The second container is an :class:`AugDicts` object. The `adb` +AugDicts object specifies the schemas to be augmented and the +augmenter configuration objects. The latter objects may be supplied +either by other Augmenter modules or from the ``augmenter`` +configuration tree on the `aug_map` supplied to the :meth:`apply` +method. + +.. autoclass:: AugmentDatabase + +.. automethod:: AugmentDatabase.apply + +.. automethod:: AugmentDatabase.from_augmap diff --git a/docs/augobjects.rst b/docs/augobjects.rst new file mode 100644 index 0000000..c38ead0 --- /dev/null +++ b/docs/augobjects.rst @@ -0,0 +1,42 @@ +Augmentation Objects +==================== + +These objects are defined in the `aug_map` argument to the `apply` +method of :class:`~pyrseas.augmentdb.AugmentDatabase`. They tie the +desired augmentations, e.g., audit columns, to the tables to be +affected, and the schemas owning the tables. + +.. module:: pyrseas.augment.schema + +Augmentation Schema +------------------- + +.. autoclass:: AugSchema + +.. automethod:: AugSchema.apply + +.. autoclass:: AugSchemaDict + +.. automethod:: AugSchemaDict.from_map + +.. automethod:: AugSchemaDict.link_current + +.. automethod:: AugSchemaDict.link_refs + + +.. module:: pyrseas.augment.table + +Augmentation Table +------------------ + +.. autoclass:: AugDbClass + +.. autoclass:: AugTable + +.. automethod:: AugTable.apply + +.. autoclass:: AugClassDict + +.. automethod:: AugClassDict.from_map + +.. automethod:: AugClassDict.link_current diff --git a/docs/cast.rst b/docs/cast.rst new file mode 100644 index 0000000..b52f794 --- /dev/null +++ b/docs/cast.rst @@ -0,0 +1,39 @@ +Casts +===== + +.. module:: pyrseas.dbobject.cast + +The :mod:`cast` module defines two classes, :class:`Cast` and +:class:`CastDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Cast +---- + +:class:`Cast` is derived from :class:`~pyrseas.dbobject.DbObject` and +represents a `Postgres cast +`_. + +A cast is identified externally as ``cast ( AS +)``. + +.. autoclass:: Cast + +.. automethod:: Cast.extern_key + +.. automethod:: Cast.identifier + +.. automethod:: Cast.to_map + +.. automethod:: Cast.create + +Cast Dictionary +--------------- + +:class:`CastDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of casts in a database. + +.. autoclass:: CastDict + +.. automethod:: CastDict.from_map diff --git a/docs/cfgobjects.rst b/docs/cfgobjects.rst new file mode 100644 index 0000000..81f0367 --- /dev/null +++ b/docs/cfgobjects.rst @@ -0,0 +1,97 @@ +Augmenter Configuration Objects +=============================== + +These configuration objects are predefined in the Augmenter modules or +can be defined or overridden by configuration elements in the +``augmenter`` map. Please see also :doc:`configitems` and +:doc:`predefaug`. + +.. module:: pyrseas.augment.function + +Configuration Functions +----------------------- + +A :class:`CfgFunction` class specifies a Postgres function to be used +by other augmenter objects. For example, this includes procedures to +be invoked by triggers used to maintain audit columns. The +:class:`CfgFunctionDict` class holds all the :class:`CfgFunction` +objects, indexed by the function name and its arguments. A +:class:`CfgFunctionSource` class represents the source code for a +function or part of that source code. A :class:`CfgFunctionTemplate` +class represents the source code for a function, which may include +other elements that can be substituted in the final result. The class +:class:`CfgFunctionSourceDict` holds all the templates currently +defined. + +.. autoclass:: CfgFunction + +.. automethod:: CfgFunction.apply + +.. autoclass:: CfgFunctionDict + +.. automethod:: CfgFunctionDict.from_map + +.. autoclass:: CfgFunctionSource + +.. autoclass:: CfgFunctionTemplate + +.. autoclass:: CfgFunctionSourceDict + + +.. module:: pyrseas.augment.column + +Configuration Columns +--------------------- + +A :class:`CfgColumn` class defines a column to be added to a table by +other augmenter objects. For example, this includes various columns +that serve to capture audit trail information. The columns can be +combined in various ways by the :class:`CfgAuditColumn` objects. The +:class:`CfgColumnDict` class holds all the :class:`CfgColumn` objects, +indexed by column name. + +.. autoclass:: CfgColumn + +.. automethod:: CfgColumn.apply + +.. autoclass:: CfgColumnDict + +.. automethod:: CfgColumnDict.from_map + + +.. module:: pyrseas.augment.trigger + +Configuration Triggers +---------------------- + +A :class:`CfgTrigger` class defines a trigger to be added to a table +by other augmentation objects. For example, this includes triggers to +maintain audit trail columns. The :class:`CfgTriggerDict` class holds +all the :class:`CfgTrigger` objects, indexed by trigger name. + +.. autoclass:: CfgTrigger + +.. automethod:: CfgTrigger.apply + +.. autoclass:: CfgTriggerDict + +.. automethod:: CfgTriggerDict.from_map + + +.. module:: pyrseas.augment.audit + +Configuration Audit Columns +--------------------------- + +A :class:`CfgAuditColumn` class defines a set of attributes (columns, +triggers) to be added to a table. The :class:`CfgAuditColumnDict` +class holds all the :class:`CfgAuditColumn` objects, indexed by +augmentation name. + +.. autoclass:: CfgAuditColumn + +.. automethod:: CfgAuditColumn.apply + +.. autoclass:: CfgAuditColumnDict + +.. automethod:: CfgAuditColumnDict.from_map diff --git a/docs/cmdargs.rst b/docs/cmdargs.rst new file mode 100644 index 0000000..3cdfca2 --- /dev/null +++ b/docs/cmdargs.rst @@ -0,0 +1,91 @@ +Common Command Line Options +=========================== + +The Pyrseas utilities support the following command line options: + +.. cmdoption:: -c + --config + + Specifies an additional `configuration file` to be read and merged + with configuration information from other sources. See + :doc:`config` for more details. + +.. cmdoption:: -H + --host + + Specifies the `host name` of the machine on which the Postgres + server is running. The default host name is determined by + Postgres (normally, a Unix-domain socket or ``localhost``). + +.. cmdoption:: -h, --help + + Show help about the program's command line arguments, and exit. + +.. cmdoption:: -o + --output + + Send output to the specified `file`. If this is omitted, the + standard output is used. + +.. cmdoption:: -p + --port + + Specifies the `port` on which the Postgres server is listening + for connections. The default port number is determined by + Postgres (normally, 5432). + +.. cmdoption:: -r + --repository + + Specifies the `path` to a directory where metadata and static data + files will be written to or read from, or where an additional + configuration file can be found. Normally, this will be the root + of a version control repository. If this is not specified on the + command line or in a configuration file, it defaults to the + current working directory. + +.. cmdoption:: -U + --user + + Postgres `user name` to connect as. The default user name is + determined by Postgres (normally, the name of the operating system + user running the program). + +.. cmdoption:: --version + + Print the program name and version identifier and exit. + +.. cmdoption:: -W, --password + + Force the program to prompt for a password before connecting to a + database. If this option is not specified and password + authentication is required, the program will resort to libpq + defaults, i.e., `password file + `_ + or `PGPASSWORD environment variable + `_. + +Short options (those only one character long) can be concatenated with +their value arguments, e.g.:: + + dbtoyaml -p5433 dbname + +Several short options can be joined together, using only a single - +prefix, as long as only the last option (or none of them) requires a +value. + +Long options (those with names longer than a single-character) can be +separated from their arguments by a '=' or passed as two separate +arguments. For example:: + + dbtoyaml --port=5433 dbname + +or:: + + dbtoyaml --port 5433 dbname + +Long options can be abbreviated as long as the abbreviation is +unambiguous:: + + dbtoyaml --pass dbname + diff --git a/docs/collation.rst b/docs/collation.rst new file mode 100644 index 0000000..1071180 --- /dev/null +++ b/docs/collation.rst @@ -0,0 +1,31 @@ +Collations +========== + +.. module:: pyrseas.dbobject.collation + +The :mod:`collation` module defines two classes, :class:`Collation` +and :class:`CollationDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Collation +--------- + +:class:`Collation` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +collation +`_. + +.. autoclass:: Collation + +.. automethod:: Collation.create + +Collation Dictionary +-------------------- + +:class:`CollationDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of collations in a database. + +.. autoclass:: CollationDict + +.. automethod:: CollationDict.from_map diff --git a/docs/column.rst b/docs/column.rst new file mode 100644 index 0000000..6b3f4bc --- /dev/null +++ b/docs/column.rst @@ -0,0 +1,52 @@ +Columns +======= + +.. module:: pyrseas.dbobject.column + +The :mod:`column` module defines two classes, :class:`Column` derived +from :class:`DbSchemaObject` and :class:`ColumnDict`, derived from +:class:`DbObjectDict`. + +Column +------ + +:class:`Column` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a column of a +table, or an attribute of a composite type. Its :attr:`keylist` +attributes are the schema name and the table name. + +A :class:`Column` has the following attributes: :attr:`name`, +:attr:`type`, :attr:`not_null`, :attr:`default` and +:attr:`collation`. The :attr:`number` attribute is also present but is +not made visible externally. + +.. autoclass:: Column + +.. automethod:: Column.to_map + +.. automethod:: Column.add + +.. automethod:: Column.add_privs + +.. automethod:: Column.diff_privileges + +.. automethod:: Column.comment + +.. automethod:: Column.drop + +.. automethod:: Column.rename + +.. automethod:: Column.alter + +Column Dictionary +----------------- + +Class :class:`ColumnDict` is a dictionary derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of columns in a database, across multiple tables. It is indexed by the +schema name and table name, and each value is a list of +:class:`Column` objects. + +.. autoclass:: ColumnDict + +.. automethod:: ColumnDict.from_map diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..7f3f739 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# +# Pyrseas documentation build configuration file, created by +# sphinx-quickstart on Fri Dec 17 22:06:15 2010. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +# -- General configuration ----------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = "Pyrseas" +copyright = "2010-2022, Joe Abbate" + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +import re +from pyrseas import __version__ as release +version = re.match(r'\d+\.\d+(?:\.\d+)?', release).group() +# The full version, including alpha/beta/rc tags. +if 'dev' in release: + release = release[:release.find('dev') + 3] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of documents that shouldn't be included in the build. +#unused_docs = [] + +# List of directories, relative to source directory, that shouldn't be searched +# for source files. +exclude_trees = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. Major themes that come with +# Sphinx are currently 'default' and 'sphinxdoc'. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_use_modindex = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = '' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Pyrseasdoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +# The paper size ('letter' or 'a4'). +#latex_paper_size = 'letter' + +# The font size ('10pt', '11pt' or '12pt'). +#latex_font_size = '10pt' + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'Pyrseas.tex', u'Pyrseas Documentation', + u'Joe Abbate', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# Additional stuff for the LaTeX preamble. +#latex_preamble = '' + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_use_modindex = True diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 0000000..fb649d9 --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,88 @@ +Configuration +============= + +The Pyrseas utilities allow you to configure various options through a +number of YAML specification files, none of which are required--but +the system configuration file is provided by the normal installation. + +If a configuration parameter is specified in more than one file, the +latter file in the list of files below overrides any earlier +specification. Any configuration item specified on the command line +takes precedence over any such item in a configuration file. + +Configuration File Name +----------------------- + +The default configuration file name is ``config.yaml``. If desired, +you can override this with the environment variable +``PYRSEAS_CONFIG_FILE``, but be aware that this will affect all three +levels below. + +System Configuration +-------------------- + +The system configuration file is distributed with Pyrseas and is +normally installed in the ``pyrseas`` library directory. + +If desired, you can override this using the ``PYRSEAS_SYS_CONFIG`` +environment variable. This can be defined as a full path, including a +file name, or a directory location, in which case the default file +name as mentioned above under `Configuration File Name`_ will be +appended to the path. + +Currently, this file includes specifications for functions, triggers +and other objects used by the :program:`dbaugment` utility. It also +includes the default directory path for storing multiple YAML files in +a VCS repository, and the path to data files for use by the data +import and export facilities. + +User Configuration +------------------ + +Each user can have his or her own configuration file. The default +location for this depends on the platform. Under Linux, BSD, OS/X and +other Unix variants, place the file under your home directory, in the +subdirectory ``.config/pyrseas/``. Under Windows, put the file in +``%APPDATA%\pyrseas\``. + +You can override the location of the user configuration file using the +``PYRSEAS_USER_CONFIG`` environment variable. This can be defined as +a full path, including a file name, or a directory location, in which +case the default file name as mentioned above under `Configuration +File Name`_ will be appended to the path. + +If present, the user configuration file will be merged with the system +configuration. + +It is recommended that the user configuration file only be used for +non-project-specific purposes. For example, if you frequently use +Pyrseas against a remote database or on a non-standard port, you can +specify the host or port in your personal configuration file. + +Repository Configuration +------------------------ + +A configuration file can be placed in a version control repository or +project directory, so that it can be under version control together +with other Pyrseas files such as the output from ``dbtoyaml +--multiple-files``. The default location for the repository can be +specified in the user configuration, using the keys ``repository`` and +``path``, for example:: + + repository: + path: /home/user/project/repo + +You can also use the :option:`--repository` command line option to +specify (or override) the directory path to the root of the repository +and the utilities will look for a configuration file in that location. + +If present, the repository configuration file will be merged with the +system and user configuration information. + +Command Line Configuration +-------------------------- + +The utilities also allow you to specify a fourth configuration file on +the command line, using the :option:`--config` command line option. +Again, if the file exists, its information will be merged with +previously read files. diff --git a/docs/configitems.rst b/docs/configitems.rst new file mode 100644 index 0000000..127f283 --- /dev/null +++ b/docs/configitems.rst @@ -0,0 +1,175 @@ +Configuration Items +=================== + +The following lists the various sections allowed in a configuration +file and the items that are recognized by the Pyrseas utilities. + +Augmenter +--------- + +This section is used by the :program:`dbaugment` utility (see +:doc:`dbaugment`). Most of these are specified in the system +configuration file delivered with Pyrseas, but can also be included or +overridden in user or repository configuration files. + +- audit_columns: This section defines combinations of columns and + triggers to be added to tables. Both columns and triggers are + specified as YAML lists (to be consistent with :program:`dbtoyaml` + YAML output), although normally a single trigger will be necessary + per column combination. The columns and triggers should reference + previously defined items in the ``columns`` and ``triggers`` + sections (see below). See :doc:`predefaug` for audit columns + defined in the system ``config.yaml``. + +- columns: This section defines prototype columns to be added to a + table by Augmenter. For each column, a valid `Postgres data type + `_ + should be included. + + You can also add a ``not_null`` constraint and a ``default`` + specification. See :doc:`predefaug` for columns defined in the + system ``config.yaml``. In a repository or user configuration file, + you can also specify an alternate name for a previously defined + column. For example, if you prefer that the ``modified_timestamp`` + columns be named ``last_update``, you can add the following to a + configuration file:: + + augmenter: + columns: + modified_timestamp: + name: last_update + +- function_templates: This section defines the source text for the + trigger functions (see below) using a template language. Any text + enclosed in double braces, e.g., ``{{modified_by_user}}``, will be + replaced, typically by a previously defined column or its alternate + name (see above). + +- functions: This section defines prototype trigger functions to be + invoked by audit columns or other augmentations. The following + items can be specified for each function: + + - description: Text for a `COMMENT + `_ + statement on the function. + + - language: Procedural language, e.g., ``plpgsql``, in which the + function is written. + + - returns: Value should be ``trigger``. + + - security_definer: Indicates whether the function is to be executed + with the privileges of the user that created it. This is usually + needed for audit column trigger functions. + + - source: This is usually a reference to a function template (see + above) enclosed in double braces, e.g., + ``{{functempl_audit_default}}``. However, in user or repository + configurations, this can also be the actual text of the function. + + See :doc:`predefaug` for functions defined in the system + configuration file. + +- schema pyrseas: This section currently defines three functions that + may be installed in the ``pyrseas`` schema if the ``full`` audit + columns specifications is added for Augmenter processing. + +- schemas and tables: Multiple ``schema schema-name`` sections can be + present, typically in a repository configuration file. Each such + section can include ``table table-name`` items, and under each the + ``audit_columns`` specifications to be added to the given table. + For example:: + + augmenter: + schema public: + table t1: + audit_columns: default + +- triggers: This section defines the prototype triggers to be used + with audit columns and other augmentations. The following items can + be specified for each trigger: + + - events: This is a list that can include one or more of ``insert``, + ``update`` or ``delete`` (the latter is not used for audit columns + but may be used in future augmentations). + + - level: This can take the values ``row`` or ``statement`` (usually + the former). + + - name: This specifies the name to be given to a trigger. It can be + a template using ``{{table_name}}`` which will then be replaced + with the actual table name on which the trigger will act. + + - procedure: This is the invocation name, e.g., ``audit_default()`` + of the function to be called when the trigger fires. + + - timing: This can take the values ``before`` or ``after`` (usually + the former). + +Database +-------- + +This section is primarily for a user configuration file. If you +frequently connect to a particular host, port or as a given user, that +are *not* the Postgres defaults, adding corresponding entries to your +user configuration file allows you to automatically override the +defaults. If for a given invocation you need to connect to or as a +different host, port or user, you can still override the configuration +using the command line options (see :doc:`cmdargs`): + +- host: Name of the host to connect. Please refer to the `Postgres + connection host documentation + `_ + for details and defaults. + +- port: Port number to connect to. See the `Postgres connection port + documentation + `_ + for more. + +- username: Name of the user to connect as. View the `Postgres + connection user documentation + `_ + for more. + +Datacopy +-------- + +This section is normally in a user or repository configuration file. +It is used by :program:`dbtoyaml` and :program:`yamltodb` to determine +which tables should be exported from or imported to the database. It +consists of schema names, using the format `schema schema_name`, +followed by lists of table names. For example:: + + datacopy: + schema public: + - t1 + - t2 + schema s1: + - t3 + +Repository +---------- + +This section is used by all utilities (but :program:`dbaugment` does +not fully support it). The "repository" is intended to be a version +control, e.g., Git, Mercurial, or Subversion, repository. + +- data: Path, relative to the root of the repository, where + :program:`dbtoyaml` and :program:`yamltodb` place or expect the + files containing data exported from or imported to the database. The + tables to be exported or imported are specified in the ``Datacopy`` + section. The default value (defined in the system ``config.yaml``) + is **metadata**. + +- metadata: Path, relative to the root of the repository, where + :program:`dbtoyaml` and :program:`yamltodb` place or expect the YAML + specification files for the database objects when the + `--multiple-files` option is used. The default value (defined + in the system ``config.yaml``) is **metadata**. + +- path: Absolute path to the root of the repository. This should + normally be specified in a user configuration file, or in a file + given with the :option:`--config` option. If not specified, this + defaults to the current working directory from which the utility is + run. diff --git a/docs/constraint.rst b/docs/constraint.rst new file mode 100644 index 0000000..8e19147 --- /dev/null +++ b/docs/constraint.rst @@ -0,0 +1,109 @@ +Constraints +=========== + +.. module:: pyrseas.dbobject.constraint + +The :mod:`constraint` module defines six classes: :class:`Constraint` +derived from :class:`DbSchemaObject`, classes +:class:`CheckConstraint`, :class:`PrimaryKey`, :class:`ForeignKey` and +:class:`UniqueConstraint` derived from :class:`Constraint`, and +:class:`ConstraintDict` derived from :class:`DbObjectDict`. + +Constraint +---------- + +Class :class:`Constraint` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a constraint +on a database table. Its :attr:`keylist` attributes are the schema +name, the table name and the constraint name. + +.. autoclass:: Constraint + +.. automethod:: Constraint.key_columns + +.. automethod:: Constraint.add + +.. automethod:: Constraint.drop + +.. automethod:: Constraint.comment + +Check Constraint +---------------- + +:class:`CheckConstraint` is derived from :class:`Constraint` and represents +a CHECK constraint. + +.. autoclass:: CheckConstraint + +.. automethod:: CheckConstraint.to_map + +.. automethod:: CheckConstraint.add + +.. automethod:: CheckConstraint.drop + +.. automethod:: CheckConstraint.alter + +Primary Key +----------- + +:class:`PrimaryKey` is derived from :class:`Constraint` and represents +a primary key constraint. + +.. autoclass:: PrimaryKey + +.. automethod:: PrimaryKey.to_map + +.. automethod:: PrimaryKey.alter + +Foreign Key +----------- + +:class:`ForeignKey` is derived from :class:`Constraint` and represents +a foreign key constraint. + +The following shows a foreign key segment of a map returned by +:meth:`to_map` and expected as argument by +:meth:`ConstraintDict.from_map` exemplifying various possibilities:: + + {'t1_fgn_key1': + { + 'columns': ['c2', 'c3'], + 'on_delete': 'restrict', + 'on_update': 'set null', + 'references': + {'columns': ['pc2', 'pc1'], 'schema': 's1', 'table': 't2'} + } + } + +.. autoclass:: ForeignKey + +.. automethod:: ForeignKey.ref_columns + +.. automethod:: ForeignKey.to_map + +.. automethod:: ForeignKey.add + +.. automethod:: ForeignKey.alter + +Unique Constraint +----------------- + +:class:`UniqueConstraint` is derived from :class:`Constraint` and +represents a UNIQUE, non-primary key constraint. + +.. autoclass:: UniqueConstraint + +.. automethod:: UniqueConstraint.to_map + +.. automethod:: UniqueConstraint.alter + +Constraint Dictionary +--------------------- + +Class :class:`ConstraintDict` is a dictionary derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of constraints in a database. + +.. autoclass:: ConstraintDict + +.. automethod:: ConstraintDict.from_map diff --git a/docs/conversion.rst b/docs/conversion.rst new file mode 100644 index 0000000..1cc4f56 --- /dev/null +++ b/docs/conversion.rst @@ -0,0 +1,33 @@ +Conversions +=========== + +.. module:: pyrseas.dbobject.conversion + +The :mod:`conversion` module defines two classes, :class:`Conversion` +and :class:`ConversionDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Conversion +---------- + +:class:`Conversion` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +conversion between character set encodings +`_. + +.. autoclass:: Conversion + +.. automethod:: Conversion.to_map + +.. automethod:: Conversion.create + +Conversion Dictionary +--------------------- + +:class:`ConversionDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of conversions in a database. + +.. autoclass:: ConversionDict + +.. automethod:: ConversionDict.from_map diff --git a/docs/database.rst b/docs/database.rst new file mode 100644 index 0000000..0ca813c --- /dev/null +++ b/docs/database.rst @@ -0,0 +1,90 @@ +Databases +========= + +.. module:: pyrseas.database + +The :mod:`database` module defines class :class:`Database`. + +Database +-------- + +A :class:`Database` can be viewed as a tree of database objects. The +tree may have one or two main branches. A tree with one main branch +is used by :program:`dbtoyaml` to hold the representation of the +database, as read from the Postgres catalogs. :program:`yamltodb` +uses a second main branch to hold the representation as read from the +YAML input specification. + +Each main branch consists of multiple subtrees for different kinds of +objects. For example, the Schemas (Postgres namespaces) subtree has +all the Postgres schema objects, the Procedures subtree has all the +Postgres functions and aggregates. The objects in the subtrees are +connected in implicit or explicit manners to related objects. For +example, the objects in the ``schema public`` are implicitly +accessible from the corresponding :class:`Schema` object because they +all share ``public`` as the first part of their internal key (see +:meth:`DbObject.key`). As another example, a table has explicit +links to constraints and indexes defined on it. + +A :class:`Database` is initialized from a +:class:`~pyrseas.database.CatDbConnection` object (a specialized class +derived from :class:`~pyrseas.lib.dbconn.DbConnection`). It consists of +one or two :class:`Dicts` (the main branches in the above +discussion). A :class:`Dicts` object holds various dictionary objects +derived from :class:`~pyrseas.dbobject.DbObjectDict`, e.g., +:class:`~pyrseas.dbobject.schema.SchemaDict`, +:class:`~pyrseas.dbobject.table.ClassDict`, and +:class:`~pyrseas.dbobject.column.ColumnDict`. The key for each +dictionary is a Python tuple (or a single value in the case of +:class:`SchemaDict` and other non-schema objects). For example, the +:class:`~pyrseas.dbobject.table.ClassDict` dictionary is indexed by +(`schema name`, `table name`)--in this context `table name` may +actually be a `sequence name`, a `view name` or a `materialized view +name`. In addition, object instances in each dictionary are linked to +related objects in other dictionaries, e.g., columns are linked to the +tables where they belong. + +The :attr:`db` :class:`Dicts` object --always present-- instantiates +the database schemas, including their tables and other objects, by +querying the system catalogs. The :attr:`ndb` :class:`Dicts` object +instantiates the schemas based on the :obj:`input_map` supplied to the +:meth:`diff_map` method. + +The :meth:`to_map` method returns and the :meth:`diff_map` method +takes as input, a Python dictionary (equivalent to a YAML or JSON +object) as shown below. It uses 'schema `schema_name`' as the key for +each schema. The value corresponding to each 'schema `schema_name`' is +another dictionary using 'sequences', 'tables', etc., as keys and more +dictionaries as values. For example:: + + {'schema public': + {'sequence seq1': { ... }, + 'sequence seq2': { ... }, + 'table t1': { ... }, + 'table t2': { ... }, + 'table t3': { ... }, + 'view v1': { ... } + }, + 'schema s1': { ... }, + 'schema s2': { ... } + } + +Refer to :class:`~pyrseas.dbobject.table.Sequence`, +:class:`~pyrseas.dbobject.table.Table` and +:class:`~pyrseas.dbobject.table.View` for details on the lower level +dictionaries. + +.. autoclass:: Database + +Methods :meth:`from_catalog` and :meth:`from_map` are for internal +use. Methods :meth:`to_map` and :meth:`diff_map` are the external API. + +.. automethod:: Database.from_catalog + +.. automethod:: Database.from_map + +.. automethod:: Database.map_from_dir + +.. automethod:: Database.to_map + +.. automethod:: Database.diff_map diff --git a/docs/dbaugment.rst b/docs/dbaugment.rst new file mode 100644 index 0000000..701ddc3 --- /dev/null +++ b/docs/dbaugment.rst @@ -0,0 +1,108 @@ +dbaugment - Augment a database +============================== + +Name +---- + +dbaugment -- Augment a Postgres database in predefined ways + +Synopsis +-------- + +:: + + dbaugment [option...] dbname [spec] + +Description +----------- + +:program:`dbaugment` is a utility for augmenting a Postgres database +with various standard attributes and procedures, such as automatically +maintained audit columns. The augmentations are defined in a +YAML-formatted ``spec`` file. + +The following is an example of a specification file:: + + augmenter: + columns: + modified_date: + not_null: true + type: date + schema public: + table t1: + audit_columns: default + table t3: + audit_columns: modified_only + +The specification file lists each schema, and within it, each table to +be augmented. Under each table the following values are currently +recognized: + + - audit_columns: This indicates that audit trail columns are to be + added to the table, e.g., a timestamp column recording when a row + was last modified. + +The first section of the specification file, under the ``augmenter`` +header, lists configuration information. This is in addition to the +built-in configuration objects (see :ref:`predef-aug`). + +:program:`dbaugment` first reads the database catalogs. It also +initializes itself from predefined configuration information. +:program:`dbaugment` then reads the specification file, which may +include additional configuration objects, and outputs a YAML file, +including the existing catalog information together with the desired +enhancements. The YAML file is suitable for input to +:program:`yamltodb` to generate the SQL statements to implement the +changes. + +Options +------- + +:program:`dbaugment` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +**dbname** + + Specifies the name of the database whose schema is to augmented. + +**spec** + + Location of the file with the augmenter specifications. If this + is omitted, the specification is read from the program's standard + input. + +Examples +-------- + +To augment a database called ``moviesdb`` according to the +specifications in the file ``movies.yaml``:: + + dbaugment moviesdb movies.yaml + +To add a column named ``updated`` to table ``public.film`` to hold the +date and time each row was inserted or updated, create a YAML +specification file, say ``film.yaml`` as follows:: + + augmenter: + columns: + modified_timestamp: + name: updated + schema public: + table film: + audit_columns: modified_only + +The first four lines configure the predefined ``modified_timestamp`` +audit column to use the name ``updated`` instead. The last three +lines direct ``dbaugment`` to apply the predefined ``modified_only`` +audit column to the ``film`` table. + +Then run the following command to generate the resulting database +specification, alter the table and create the needed trigger and +function. + + dbaugment moviesdb film.yaml | yamltodb moviesdb -u + +See Also +-------- + + :ref:`predef-aug` diff --git a/docs/dbobject.rst b/docs/dbobject.rst new file mode 100644 index 0000000..db126f3 --- /dev/null +++ b/docs/dbobject.rst @@ -0,0 +1,101 @@ +Database Objects +================ + +.. module:: pyrseas.dbobject + +The :mod:`dbobject` module defines two low-level classes and an +intermediate class. Most Pyrseas classes are derived from either +:class:`DbObject` or :class:`DbObjectDict`. + +Database Object +--------------- + +A :class:`DbObject` represents a database object such as a schema, +table, or column, defined in a Postgres `system catalog +`_. It is +initialized from a dictionary of attributes. Derived classes should +define a :attr:`keylist` that is a list of attribute names that +uniquely identify each object instance within the database. + +.. autoclass:: DbObject + +.. autoattribute:: DbObject.objtype + +.. autoattribute:: DbObject.keylist + +.. automethod:: DbObject.key + +The following methods are generally used to map objects for external +output: + +.. automethod:: DbObject.extern_key + +.. automethod:: DbObject.query + +.. automethod:: DbObject.extern_filename + +.. automethod:: DbObject.identifier + +.. automethod:: DbObject.to_map + +.. automethod:: DbObject.map_privs + +The following methods generate SQL statements from the object +properties and sometimes from a second object: + +.. automethod:: DbObject.comment + +.. automethod:: DbObject.alter_owner + +.. automethod:: DbObject.drop + +.. automethod:: DbObject.rename + +.. automethod:: DbObject.alter + +.. automethod:: DbObject.diff_privileges + +.. automethod:: DbObject.diff_description + + +Database Object Dictionary +-------------------------- + +A :class:`DbObjectDict` represents a collection of :class:`DbObject`'s +and is derived from the Python built-in type :class:`dict`. If a +:class:`~pyrseas.lib.dbconn.DbConnection` object is used for +initialization, an internal method is called to initialize the +dictionary from the database catalogs. The :class:`DbObjectDict` +:meth:`fetch` method fetches all objects using the `query` method +defined by derived classes. Derived classes should also define a +:attr:`cls` attribute for the associated :class:`DbObject` class, +e.g., :class:`~pyrseas.schema.SchemaDict` sets :attr:`cls` to +:class:`~pyrseas.schema.Schema`. + +.. autoclass:: DbObjectDict + +.. autoattribute:: DbObjectDict.cls + +.. automethod:: DbObjectDict.to_map + +.. automethod:: DbObjectDict.fetch + + +Schema Object +------------- + +A :class:`DbSchemaObject` is derived from :class:`DbObject`. It is +used as a base class for objects owned by a schema and to define +certain common methods. This is different from the +:class:`~pyrseas.schema.Schema` that represents the schema (Postgres +namespace) itself. + +.. autoclass:: DbSchemaObject + +.. automethod:: DbSchemaObject.identifier + +.. automethod:: DbSchemaObject.qualname + +.. automethod:: DbSchemaObject.unqualify + +.. automethod:: DbSchemaObject.rename diff --git a/docs/dbtoyaml.rst b/docs/dbtoyaml.rst new file mode 100644 index 0000000..9212996 --- /dev/null +++ b/docs/dbtoyaml.rst @@ -0,0 +1,293 @@ +dbtoyaml - Database to YAML +=========================== + +Name +---- + +dbtoyaml -- extract the schema of a Postgres database in YAML format + +Synopsis +-------- + +:: + + dbtoyaml [option...] dbname + +Description +----------- + +:program:`dbtoyaml` is a utility for extracting the schema of a +Postgres database to a `YAML `_ formatted +specification. By default, the specification is output as a single +output stream, which can be redirected or explicitly sent to a file. +As an alternative, the ``--multiple-files`` option allows you to break +down the specification into multiple files, in general, one for each +object (see `Multiple File Output`_). + +Note that `JSON `_ is an official +subset of YAML version 1.2, so the :program:`dbtoyaml` output should +also be compatible with JSON tools. + +A sample of the output format is as follows:: + + schema public: + owner: postgres + privileges: + - postgres: + - all + - PUBLIC: + - all + table t1: + check_constraints: + t1_c2_check: + columns: + - c2 + expression: (c2 > 123) + columns: + - c1: + not_null: true + type: integer + - c2: + type: smallint + - c3: + default: 'false' + type: boolean + - c4: + type: text + foreign_keys: + t1_c2_fkey: + columns: + - c2 + references: + columns: + - c21 + schema: s1 + table: t2 + owner: alice + primary_key: + t1_pkey: + columns: + - c1 + schema s1: + owner: bob + privileges: + - bob: + - all + - alice: + - all + table t2: + columns: + - c21: + not_null: true + type: integer + - c22: + type: character varying(16) + owner: bob + primary_key: + t2_pkey: + columns: + - c21 + privileges: + - bob: + - all + - PUBLIC: + - select + - alice: + - insert: + grantable: true + - delete: + grantable: true + - update: + grantable: true + - carol: + grantor: alice + privs: + - insert + + +The above should be mostly self-explanatory. The example database has +two tables, named ``t1`` and ``t2``, the first --owned by user +'alice'-- in the ``public`` schema and the second --owned by user +'bob'-- in a schema named ``s1`` (also owned by 'bob'). +The ``columns:`` specifications directly under each table list each +column in that table, in the same order as shown by Postgres. The +specifications ``primary_key:``, ``foreign_keys:`` and +``check_constraints:`` define PRIMARY KEY, FOREIGN KEY and CHECK +constraints for a given table. Additional specifications (not shown) +define unique constraints and indexes. + +User 'bob' has granted all privileges to 'alice' on the ``s1`` schema. +On table ``t2``, he also granted SELECT to PUBLIC; INSERT, UPDATE and +DELETE to 'alice' with GRANT OPTION; and she has in turn granted +INSERT to user 'carol'. + +:program:`dbtoyaml` currently supports extracting information about +nearly all types of Postgres database objects. See :ref:`api-ref` +for a list of supported objects. + +The behavior and options of ``dbtoyaml`` are patterned after the +`pg_dump utility +`_ +since it is most analogous to using ``pg_dump --schema-only``. + +Multiple File Output +-------------------- + +.. program:: dbtoyaml + +The :option:`--multiple-files` option breaks down the output into +multiple files under a given root directory. The root is created if +it does not exist. The root directory name defaults to ``metadata`` +in the system configuration file. The location of the root directory +defaults to the configuration item ``repository.path`` or can be +specified using the `--repository` option (see :doc:`config` +and :doc:`cmdargs` for further details). + +The first level contains ``schema.`` subdirectories, +``schema..yaml`` files and ``..yaml`` files, +where ```` is the name of the corresponding objects and +```` is the type of top-level (non-schema) object. Note that +non-schema refers to Postgres extensions, casts, languages or +foreign data wrappers. + +The second level, i.e., the ``schema.`` subdirectories contain +``..yaml`` files for each object in the particular +schema (but see below for caveats). + +Object Name Conflicts +~~~~~~~~~~~~~~~~~~~~~ + +The names of Postgres objects can include characters that are not +allowed in filesystem object names. The most common example is the +division operator ('/'), but even table names can include +non-alphanumeric characters, if the identifiers are quoted. + +In addition, one can define two or more objects with the same base +name, e.g., function ``foo(integer)`` and function ``foo(text)``, or a +table named ``"My Table"`` and another named ``"my table"`` or +``"MY TABLE"``. On certain operating systems, i.e., Windows, it is not +possible to create two files in the same directory that differ only in +the case of their characters. + +In order to deal with the aforementioned issues, ``dbtoyaml`` places +certain objects in common files and transforms object identifiers so +that they are suitable for use in files and directories. For example, +the information for all user-defined casts are written to the file +``cast.yaml`` in the root directory. Functions with the same name but +different arguments are written to a single file, e.g., +``function.foo.yaml`` in the first example above. Identifiers are +also converted to all lowercase, non-alphanumeric characters +(excluding underscore) are converted to underscores and, by default, +schema object names are truncated to 32 characters. + +If two object names, thus transformed, map to the same string, then +the objects' information is written to the same file, e.g., +``table.my_table.yaml`` in the second example above. If you prefer to +change the default truncation length, please define the environment +variable ``PYRSEAS_MAX_IDENT_LEN`` to some integer value (up to 63). + +Version Control and Dropped Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It is expected that the output of ``dbtoyaml --multiple-files`` will +be placed under version control. Further invocations should then +update the files in the same directory tree. However, if an object is +dropped from the database ``dbtoyaml`` would normally only output +files for new or changed objects--and thus keep the dropped object +file under version control. To deal with dropped objects, ``dbtoyaml +-m`` outputs a special YAML "index" file, named +``database..yaml`` in the root directory. When ``dbtoyaml +-m`` is run a second time, it looks for this "index" file and if +found, proceeds to delete the previous run's ``.yaml`` files before +outputting new ones. + +Options +------- + +:program:`dbtoyaml` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +dbname + + Specifies the name of the database whose schema is to be extracted. + +.. cmdoption:: -m, --multiple-files + + Extracts the schema to a two-level directory tree. See `Multiple + File Output`_ above. + +.. cmdoption:: -n + --schema + + Extracts only a schema matching `schema`. By default, all schemas + are extracted. Multiple schemas can be extracted by using multiple + ``-n`` switches. Note that normally all objects that belong to the + schema are extracted as well, unless excluded otherwise. + +.. cmdoption:: -N + --exclude-schema + + Does not extract schema matching `schema`. This can be given more + than once to exclude several schemas. + +.. cmdoption:: -O, --no-owner + + Do not output object ownership information. By default, as seen + in the sample output above, database objects (schemas, tables, + etc.) that can be owned by some user, are shown with an "owner: + *username*" element. The :option:`-O` switch suppresses all those + lines. + + NOTE: If you specify `--no-owner`, you will most likely also want + to specify :option:`--no-privileges`. If the former is used + without the latter the resulting YAML output will have privilege + information without user data, which will cause errors if the YAML + is then fed to :doc:`yamltodb`. + +.. cmdoption:: -t + --table
+ + Extract only tables matching `table`. Multiple tables can be + extracted by using multiple :option:`-t` switches. Note that + selecting a table may cause other objects, such as an owned + sequence, to be extracted as well + +.. cmdoption:: -T
+ --exclude-table
+ + Do not extract tables matching `table`. Multiple tables can be + excluded by using multiple :option:`-T` switches. + +.. cmdoption:: -x, --no-privileges + + Do not output access privilege information. By default, as seen + in the sample output above, if specific GRANTs have been issued on + various objects (schemas, tables, etc.), the privileges are shown + under each object. The :option:`-x` switch suppresses all those + lines. + + See also the NOTE under :option:`--no-owner`. + +Examples +-------- + +To extract a database called ``moviesdb`` into a file:: + + dbtoyaml moviesdb > moviesdb.yaml + +To extract only the schema named ``store``:: + + dbtoyaml --schema=store moviesdb > moviesdb.yaml + +To extract the tables named ``film`` and ``genre``:: + + dbtoyaml -t film -t genre moviesdb -o moviesdb.yaml + +To extract objects, to standard output, except those in schemas +``product`` and ``store``:: + + dbtoyaml -N product -N store moviesdb + +To extract objects to a directory under version control:: + + dbtoyaml moviesdb -m movies/dbspec diff --git a/docs/devel.rst b/docs/devel.rst new file mode 100644 index 0000000..2b26afb --- /dev/null +++ b/docs/devel.rst @@ -0,0 +1,97 @@ +.. _development: + +Development +=========== + +The following details the tools needed to contribute to the +development of Pyrseas. If you have any doubts or questions, please +open an issue on GitHub (https://github.com/perseas/Pyrseas/issues). +In addition, see *Version Control* below on how to set up a GitHub +account to participate in development. + +Requirements +------------ + +- Git + +- Python + +- Postgres + +- Psycopg3 + +- PyYAML + +- Tox + +Version Control +--------------- + +Pyrseas uses `Git `_ to control changes to its +source code. As mentioned under :ref:`download`, the master Git +`repository `_ is located at GitHub. + +To install Git, either `download and install +`_ the latest stable release for your +platform or follow the `Pro Git` `installation instructions +`_. For +most Linux users, ``apt-get`` or ``yum`` (depending on Linux flavor) +will be the simplest means to install the ``git-core`` package. For +Windows, downloading the installer and selecting ``Git Bash`` gives +you not only Git but a Bash shell, which is handy if you're coming +from a Linux/Unix background. + +Once Git is installed, change to a suitable directory and clone the +master repository:: + + git clone https://github.com/perseas/Pyrseas.git + +To be able to create a fork on GitHub, open an issue or participate in +Pyrseas development, you'll first have to `create a GitHub account +`_. + +Programming Language +-------------------- + +To contribute to Pyrseas, you need a version of `Python +`_. You can develop using Python 3.7 or higher. + +If Python is not already available on your machine, either `download +and install one or both `_ of the +production releases for your platform, follow the applicable +installation instructions given in `The Hitchhiker’s Guide to Python! +`_ or install it from your +platform's package management system. + +Database Installation +--------------------- + +To participate in Pyrseas development, you'll also need one or more +installations of `Postgres `_, versions +13, 12, 11 or 10. If you only have limited space, it is +preferable to install one of the latest two versions. + +The versions can be obtained as binary packages or installers from the +`Postgres.org website `_. The +site also includes instructions for installing from package management +systems or building it from source. + +To access Postgres from Python, you have to install the `Psycopg +`_ version 3 adapter. You can either follow the +instructions in `Psycopg's site +`_, or install it from +your package management system. + +Other Libraries and Tools +------------------------- + +The ``dbtoyaml`` and ``yamltodb`` utilities use the `PyYAML +`_ library. You can install it from +the PyYAML site, or possibly from your package management system. For +Windows 64-bit, please read the note under :ref:`installer`. + +To easily run the Pyrseas tests against various Python/Postgres +version combinations, you will need `pytest +`_ and `Tox +`_. Please refer to +:ref:`testing` for more information. diff --git a/docs/eventtrig.rst b/docs/eventtrig.rst new file mode 100644 index 0000000..71006c8 --- /dev/null +++ b/docs/eventtrig.rst @@ -0,0 +1,33 @@ +Event Triggers +============== + +.. module:: pyrseas.dbobject.eventtrig + +The :mod:`eventtrig` module defines two classes, :class:`EventTrigger` and +:class:`EventTriggerDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Event Trigger +-------------- + +:class:`EventTrigger` is derived from +:class:`~pyrseas.dbobject.DbObject` and represents an `event trigger +`_ +available from Postgres 9.3 onwards. + +.. autoclass:: EventTrigger + +.. automethod:: EventTrigger.to_map + +.. automethod:: EventTrigger.create + +Event Trigger Dictionary +------------------------ + +:class:`EventTriggerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of event triggers in a database. + +.. autoclass:: EventTriggerDict + +.. automethod:: EventTriggerDict.from_map diff --git a/docs/extension.rst b/docs/extension.rst new file mode 100644 index 0000000..6f5b3c3 --- /dev/null +++ b/docs/extension.rst @@ -0,0 +1,33 @@ +Extensions +========== + +.. module:: pyrseas.dbobject.extension + +The :mod:`extension` module defines two classes, :class:`Extension` +and :class:`ExtensionDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Extension +--------- + +:class:`Extension` is derived from +:class:`~pyrseas.dbobject.DbObject` and represents a `Postgres +extension +`_. + +.. autoclass:: Extension + +.. automethod:: Extension.create + +.. automethod:: Extension.alter + +Extension Dictionary +-------------------- + +:class:`ExtensionDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of extensions in a database. + +.. autoclass:: ExtensionDict + +.. automethod:: ExtensionDict.from_map diff --git a/docs/foreign.rst b/docs/foreign.rst new file mode 100644 index 0000000..d01792c --- /dev/null +++ b/docs/foreign.rst @@ -0,0 +1,142 @@ +Foreign Data Objects +==================== + +.. module:: pyrseas.dbobject.foreign + +The :mod:`foreign` module defines nine classes related to Postgres +foreign data wrappers (FDWs), namely: :class:`DbObjectWithOptions` +derived from :class:`DbObject`, classes :class:`ForeignDataWrapper`, +:class:`ForeignServer` and :class:`UserMapping` derived from +:class:`DbObjectWithOptions`, :class:`ForeignTable` derived from +:class:`DbObjectWithOptions` and :class:`Table`, classes +:class:`ForeignDataWrapperDict`, :class:`ForeignServerDict` and +:class:`UserMappingDict` derived from :class:`DbObjectDict`, and +:class:`ForeignTableDict` derived from :class:`ClassDict`. + +Database Object With Options +---------------------------- + +:class:`DbObjectWithOptions` is derived from +:class:`~pyrseas.dbobject.DbObject`. It is a helper class for dealing +with the OPTIONS clauses common to the foreign data objects. + +.. autoclass:: DbObjectWithOptions + +.. automethod:: DbObjectWithOptions.to_map + +.. automethod:: DbObjectWithOptions.options_clause + +.. automethod:: DbObjectWithOptions.diff_options + +.. automethod:: DbObjectWithOptions.alter + +Foreign Data Wrapper +-------------------- + +:class:`ForeignDataWrapper` is derived from `DbObjectWithOptions` and +represents a `Postgres foreign data wrapper +`_. +See also `Foreign Data +`_ +and `Writing A Foreign Data Wrapper +`_. + +.. autoclass:: ForeignDataWrapper + +.. automethod:: ForeignDataWrapper.to_map + +.. automethod:: ForeignDataWrapper.create + +Foreign Data Wrapper Dictionary +------------------------------- + +:class:`ForeignDataWrapperDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of foreign data wrappers in a database. + +.. autoclass:: ForeignDataWrapperDict + +.. automethod:: ForeignDataWrapperDict.from_map + +Foreign Server +-------------- + +:class:`ForeignServer` is derived from :class:`DbObjectWithOptions` +and represents a `Postgres foreign server +`_. + +.. autoclass:: ForeignServer + +.. automethod:: ForeignServer.identifier + +.. automethod:: ForeignServer.to_map + +.. automethod:: ForeignServer.create + +Foreign Server Dictionary +------------------------- + +:class:`ForeignServerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a Python dictionary +that represents the collection of foreign servers in a database. + +.. autoclass:: ForeignServerDict + +.. automethod:: ForeignServerDict.from_map + +.. automethod:: ForeignServerDict.to_map + +User Mapping +------------ + +:class:`UserMapping` is derived from :class:`DbObjectWithOptions` and +represents a `mapping of a Postgres user to a foreign server +`_. + +.. autoclass:: UserMapping + +.. automethod:: UserMapping.extern_key + +.. automethod:: UserMapping.identifier + +.. automethod:: UserMapping.create + +User Mapping Dictionary +----------------------- + +:class:`UserMappingDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of user mappings in a database. + +.. autoclass:: UserMappingDict + +.. automethod:: UserMappingDict.from_map + +.. automethod:: UserMappingDict.to_map + +Foreign Table +------------- + +:class:`ForeignTable` is derived from :class:`DbObjectWithOptions` and +:class:`~pyrseas.dbobject.table.Table`. It represents a `Postgres +foreign table +`_. + +.. autoclass:: ForeignTable + +.. automethod:: ForeignTable.to_map + +.. automethod:: ForeignTable.create + +.. automethod:: ForeignTable.drop + +Foreign Table Dictionary +------------------------ + +:class:`ForeignTableDict` is derived from +:class:`~pyrseas.dbobject.table.ClassDict`. It is a dictionary that +represents the collection of foreign tables in a database. + +.. autoclass:: ForeignTableDict + +.. automethod:: ForeignTableDict.from_map diff --git a/docs/function.rst b/docs/function.rst new file mode 100644 index 0000000..4b89349 --- /dev/null +++ b/docs/function.rst @@ -0,0 +1,65 @@ +Functions +========= + +.. module:: pyrseas.dbobject.function + +The :mod:`function` module defines four classes: class :class:`Proc` +derived from :class:`DbSchemaObject`, classes :class:`Function` and +:class:`Aggregate` derived from :class:`Proc`, and class +:class:`ProcDict` derived from :class:`DbObjectDict`. + +Procedure +--------- + +Class :class:`Proc` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a regular or +aggregate function. + +.. autoclass:: Proc + +.. automethod:: Proc.extern_key + +.. automethod:: Proc.identifier + +Function +-------- + +:class:`Function` is derived from :class:`Proc` and represents a +`Postgres user-defined function +`_. + + +.. autoclass:: Function + +.. automethod:: Function.to_map + +.. automethod:: Function.create + +.. automethod:: Function.alter + +.. automethod:: Function.drop + +Aggregate Function +------------------ + +:class:`Aggregate` is derived from :class:`Proc` and represents a +`Postgres user-defined aggregate function +`_. + +.. autoclass:: Aggregate + +.. automethod:: Aggregate.to_map + +.. automethod:: Aggregate.create + +Procedure Dictionary +-------------------- + +:class:`ProcDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of regular and aggregate functions in a +database. + +.. autoclass:: ProcDict + +.. automethod:: ProcDict.from_map diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..4bc7ae2 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,132 @@ +Pyrseas +======= + +Pyrseas provides utilities to describe a PostgreSQL database schema as +YAML, to verify the schema against the same or a different database +and to generate SQL that will modify the schema to match the YAML +description. + +Features +-------- + +- Outputs a YAML/JSON description of a PostgreSQL database's tables + and other objects (metadata), suitable for storing in a version + control repository + +- Generates SQL statements to modify a database so that it will match + an input YAML/JSON specification + +- Generates an augmented YAML description of a PostgreSQL database + from its catalogs and an augmentation specification. + +Requirements +------------ + +- `PostgreSQL `_ 10 or higher + +- `Python `_ 3.7 or higher + +- `Psycopg3 `_ 3.1 or higher + +- `PyYAML `_ 5.3 or higher + +Contents +-------- + +.. toctree:: + :maxdepth: 2 + + overview + install + config + configitems + devel + testing + issues + predefaug +.. toctree:: + :maxdepth: 1 + + dbaugment + dbtoyaml + yamltodb + cmdargs + +.. _api-ref: + +API Reference +------------- + +Currently, the only external APIs are the class +:class:`~pyrseas.database.Database` and the methods +:meth:`~pyrseas.database.Database.to_map` and +:meth:`~pyrseas.database.Database.diff_map` of the latter. Other +classes and methods are documented mainly for developer use. + +.. toctree:: + :maxdepth: 2 + + dbobject + database + schema + +Non-schema Objects +~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + cast + eventtrig + extension + foreign + language + +Tables and Related Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + table + column + constraint + indexes + rule + +Functions, Operators and Triggers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + function + operator + operfamily + operclass + trigger + +Types and Other Schema Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. toctree:: + :maxdepth: 2 + + collation + conversion + textsearch + type + +Augmenter API Reference +----------------------- + +.. toctree:: + :maxdepth: 2 + + augmentdb + cfgobjects + augobjects + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/indexes.rst b/docs/indexes.rst new file mode 100644 index 0000000..ed07d62 --- /dev/null +++ b/docs/indexes.rst @@ -0,0 +1,43 @@ +Indexes +======= + +.. module:: pyrseas.dbobject.index + +The :mod:`index` module defines two classes, :class:`Index` and +:class:`IndexDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Index +----- + +Class :class:`Index` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents an index on a +database table, other than a primary key or unique constraint +index. Its :attr:`keylist` attributes are the schema name, the table +name and the index name. Note that index names are supposed to be +unique with a given schema so the table name doesn't have to be part +of the :attr:`keylist`, but has been retained to facilitate certain +operations. + +.. autoclass:: Index + +.. automethod:: Index.key_expressions + +.. automethod:: Index.to_map + +.. automethod:: Index.create + +.. automethod:: Index.alter + +.. automethod:: Index.drop + +Index Dictionary +---------------- + +Class :class:`IndexDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of indexes in a database. + +.. autoclass:: IndexDict + +.. automethod:: IndexDict.from_map diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 0000000..70ba9da --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,182 @@ +Installation +============ + +Summary +------- + +For the latest release, use:: + + pip install Pyrseas + +For development:: + + git clone git://github.com/perseas/Pyrseas.git + cd Pyrseas + python setup.py install + +Requirements +------------ + +Pyrseas provides tools for `Postgres `_, +so obviously you need **Postgres** to start with. Pyrseas has been +tested with PG 10, 11, 12 and 13 and we'll certainly keep up +with future releases. Please refer to the `Postgres download page +`_ to find a distribution for the +various Linux, Unix and Windows platforms supported. + +You will also need **Python**. Pyrseas was originally developed using +`Python ` 2 and then ported to Python 3 and +tested against versions from 3.7 through 3.9. On Linux or \*BSD, +Python may already be part of your distribution or may be available as +a package. For Windows and Mac OS please refer to the `Python +download page `_ for installers and +instructions. + +Pyrseas talks to the Postgres DBMS via the **Psycopg adapter**. +Pyrseas has been tested with `psycopg `_ +3.1. Psycopg may be available as a package on most Linux or +\*BSD distributions and can also be downloaded or installed from PyPI. +Please refer to the `Psycopg download page +`_ for more details. + +.. note:: If you install Pyrseas using ``pip`` (see below) and you + have not already installed Psycopg, e.g., when installing into a + ``virtualenv`` environment created with ``--no-site-packages``, you + may need to have installed the Postgres and Python development + packages, and a C compiler, as ``pip`` may download and attempt to + build and install psycopg before installing Pyrseas. + +The Pyrseas utilities rely on **PyYAML**, a `YAML `_ +library. This may be available as a package for your operating system +or it can be downloaded from the `Python Package Index (PyPI) +`_. + +.. _download: + +Downloading +----------- + +Pyrseas is available at the following locations: + + - `Python Package Index `_ + - `Postgres Extension Network (PGXN) `_ + - `GitHub repository `_ + +You can download the distribution from PyPI in gzip-compressed tar or +ZIP archive format, but you can download *and* install it using +``Pip``. See `Python Installer`_ below for details. + +PGXN provides a ZIP archive which you can download or you can download +*and* install using the PGXN client (see `PGXN Client`_ below). + +The GitHub repository holds the Pyrseas source code, tagged according +to the various releases, e.g., v0.9.0, and including unreleased +modifications. To access it, you need `Git `_ +which is available as a package in most OS distributions or can be +downloaded from the `Git download page +`_. You can fetch the Pyrseas sources by +issuing the following command:: + + git clone https://github.com/perseas/Pyrseas.git + +This will create a ``Pyrseas`` directory tree (you can use a different +target name by adding it to the above commands). To list available +releases, change to the subdirectory and invoke ``git tag``. To +switch to a particular release, use:: + + git checkout vn.n.n + +where *vn.n.n* is the release identifier. Use ``git checkout master`` +to revert to the main (master) branch. To fetch the latest updates, +use:: + + git pull + +Installation +------------ + +Extracting Sources +~~~~~~~~~~~~~~~~~~ + +Once you have downloaded an archive from PyPI or PGXN, you need to +extract the sources. For a gzip-compressed tar file, use:: + + tar xzf Pyrseas-n.n.n.tar.gz + +where *n.n.n* is the release version. For a ZIP archive, use:: + + unzip Pyrseas-n.n.n.zip + +Both commands above will create a directory ``Pyrseas-n.n.n`` and you +will want to ``cd`` to it before proceeding with the installation. + +Installing +~~~~~~~~~~ + +If you have superuser or similar administrative privileges, you can +install Pyrseas for access by multiple users on your system. On Linux +and other Unix-flavored systems, you can install from the extracted +``Pyrseas-n.n.n`` source directory or from the root directory of the +``git`` clone, using the following command:: + + sudo python setup.py install + +That will install the :doc:`dbtoyaml ` and :doc:`yamltodb +` utility scripts in a directory such as +``/usr/local/bin``. The library sources and bytecode files will be +placed in a ``pyrseas`` subdirectory under ``site-packages`` or +``dist-packages``, e.g., +``/usr/local/lib/python3.6/dist-packages/pyrseas``. + +On Windows, from an account with Administrator privileges, you can +use:: + + python setup.py install + +That will install the Pyrseas utilities in the ``Scripts`` folder of +your Python installation. The source and bytecode files will go in +the ``site-packages`` folder, e.g., +``C:\Python37\Lib\site-packages\pyrseas``. + +.. _installer: + +Python Installer +~~~~~~~~~~~~~~~~ + +You can also download and install Pyrseas using `pip +`_. For example, on Linux do:: + + sudo pip install Pyrseas + +If this is the first time you are installing a Python package, please +do yourself a favor and read and follow the instructions in the +"Distribute & Pip" subsection of the "Installing Python on ..." +section for your platform of the `The Hitchhiker’s Guide to Python! +`_. + +.. note:: On FreeBSD, it has been reported that it is necessary to + install the Python ``distribute`` package, prior to + installing Pyrseas with ``pip``. This may also be necessary + on other BSD variants. + +.. note:: On Windows 64-bit, it has been reported that it is necessary + to obtain unofficial versions of the ``distribute`` and + ``PyYAML`` packages, available at `University of California, + Irvine `_. For + a detailed tutorial, see `this post + `_. + +``Pip`` can also be used in a Python `virtualenv +`_ environment, in which case +you *don't* need to prefix the commands with ``sudo``. + +``Pip`` also provides the ability to uninstall Pyrseas. + +PGXN Client +~~~~~~~~~~~ + +The PGXN `client `_ (available +at PyPI) can be used to download and install Pyrseas from PGXN. Usage +is:: + + pgxn install pyrseas diff --git a/docs/issues.rst b/docs/issues.rst new file mode 100644 index 0000000..97cde21 --- /dev/null +++ b/docs/issues.rst @@ -0,0 +1,114 @@ +Known Issues +============ + +The following summarizes notable deficiencies in the current release +of the Pyrseas utilities. For further details please refer to the +discussions in the `Pyrseas issue tracker +`_. Suggestions or patches +to deal with these issues are welcome. + +Coverage of Postgres Objects +---------------------------- + +An important Pyrseas objective is to support creating, altering or +dropping nearly any Postgres object accessible through SQL, including +adding, modifying or removing any attributes or features of those +objects. At present, we believe Pyrseas covers roughly over 90% of +the Postgres object/attribute universe. Please refer to the `Feature +Matrix `_ for details. + +This is a continuing effort since Postgres keeps adding new features +in each release, such as the table PARTITIONING syntax in PG 10. We +have documented current limitations in the issue tracker, see, for +example, issues `135 `_ +and `178 `_. Please +open an issue on the tracker if you find objects or features needing +additional support. + +Object Dependencies +------------------- + +The first releases of :program:`yamltodb` used a generally fixed +traversal order when generating SQL. This caused problems with +complex dependencies between objects (e.g., views that depended on +functions that depended on types). Release 0.8 introduced a +topological sort of objects based on their dependencies. The +resulting dependency graph is now used to drive SQL generation. This +should eliminate most object dependency problems seen with the +previous architecture. However, certain issues still remain. +Specifically, if an object depends on a Postgres internally-defined +object, or on an object defined by a Postgres extension, the Pyrseas +utilities may not behave as expected (see issue `175 +`_ for additional +discussion). + +Object renaming +--------------- + +Pyrseas provides support for generating SQL statements to rename +various database objects, e.g., ALTER TABLE t1 RENAME TO t2, using an +'oldname' tag which can be added to objects that support SQL RENAME. +The tag has to be added manually to a YAML specification for yamltodb +to act on it and cannot be kept in the YAML file for subsequent runs. +This is not entirely satisfactory for storing the YAML file in a +version control system. + +Memory utilization +------------------ + +The yamltodb utility compares the existing and input metadata by +constructing parallel, in-memory representations of the database +catalogs and the input YAML specification. If the database has a +large number of objects, e.g., in the thousands of tables, the +utility's memory usage may be noticeable. + + +Multiline Strings +----------------- + +The text of function source code, view definitions or object COMMENTs +present a problem when they span multiple lines. The default YAML +output format is to enclose the entire string in double quotes, to +show newlines that are part of the text as escaped characters (i.e., +``\n``) and to break the text into lines with a +backslash-newline-indentation-backslash pattern. For example:: + + source: "\n SELECT inventory_id\n FROM inventory\n WHERE film_id =\ + \ $1\n AND store_id = $2\n AND inventory_in_stock(inventory_id);\n" + +This is not very readable, but it does allow YAML to read it back and +correctly reconstruct the original string. To improve readability, +Pyrseas 0.7 introduced special processing for these strings. By using +YAML notation, the same string is represented as follows:: + + source: |2 + + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND NOT inventory_in_stock(inventory_id); + +Note also that if your function source code has trailing spaces at the +end of lines, they would normally be represented in the original +default format. However, in the interest of readability, +:program:`dbtoyaml` will remove the trailing spaces from the text. + +Index and Partitioning Expressions +---------------------------------- + +Postgres allows users to create `indexes using expressions +`_. +A user can also mix expressions with regular columns. The Postgres +catalogs store the index information in a bespoke fashion: an array of +column numbers where a zero indicates an expression and a list of +expression trees (an internal format) for the expressions, with +additional arrays for collation information, operator classes and +index options such as ``ASC`` or ``DESC``. Although the +``pg_get_indexdef`` system catalog function can be used to obtain a +full ``CREATE INDEX`` statement, Pyrseas has chosen to specify each +column or expression separately in the YAML definitions. This has +not been satisfactory in complex cases (see for example issue `170 +`_) and is an area +requiring further attention. A similar situation exists for table +partitioning using expressions. diff --git a/docs/language.rst b/docs/language.rst new file mode 100644 index 0000000..4e4542b --- /dev/null +++ b/docs/language.rst @@ -0,0 +1,35 @@ +Procedural Languages +==================== + +.. module:: pyrseas.dbobject.language + +The :mod:`language` module defines two classes, :class:`Language` and +:class:`LanguageDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Procedural Language +------------------- + +:class:`Language` is derived from :class:`~pyrseas.dbobject.DbObject` +and represents a procedural language. + + +.. autoclass:: Language + +.. automethod:: Language.to_map + +.. automethod:: Language.drop + +.. automethod:: Language.create + +Language Dictionary +------------------- + +:class:`LanguageDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of procedural languages in a +database. Internal languages ('internal', 'c' and 'sql') are excluded. + +.. autoclass:: LanguageDict + +.. automethod:: LanguageDict.from_map diff --git a/docs/operator.rst b/docs/operator.rst new file mode 100644 index 0000000..7530daa --- /dev/null +++ b/docs/operator.rst @@ -0,0 +1,36 @@ +Operators +========= + +.. module:: pyrseas.dbobject.operator + +The :mod:`operator` module defines two classes: class +:class:`Operator` derived from :class:`DbSchemaObject` and class +:class:`OperatorDict` derived from :class:`DbObjectDict`. + +Operator +--------- + +:class:`Operator` is derived from :class:`DbSchemaObject` and +represents a `Postgres user-defined operator +`_. + +.. autoclass:: Operator + +.. automethod:: Operator.extern_key + +.. automethod:: Operator.qualname + +.. automethod:: Operator.identifier + +.. automethod:: Operator.create + +Operator Dictionary +------------------- + +:class:`OperatorDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operators in a database. + +.. autoclass:: OperatorDict + +.. automethod:: OperatorDict.from_map diff --git a/docs/operclass.rst b/docs/operclass.rst new file mode 100644 index 0000000..9a53ded --- /dev/null +++ b/docs/operclass.rst @@ -0,0 +1,36 @@ +Operator Classes +================ + +.. module:: pyrseas.dbobject.operclass + +The :mod:`operclass` module defines two classes: class +:class:`OperatorClass` derived from :class:`DbSchemaObject` and class +:class:`OperatorClassDict` derived from :class:`DbObjectDict`. + +Operator Class +-------------- + +:class:`OperatorClass` is derived from :class:`DbSchemaObject` and +represents a `Postgres operator class +`_. + +.. autoclass:: OperatorClass + +.. automethod:: OperatorClass.extern_key + +.. automethod:: OperatorClass.identifier + +.. automethod:: OperatorClass.to_map + +.. automethod:: OperatorClass.create + +Operator Class Dictionary +------------------------- + +:class:`OperatorClassDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operator classes in a database. + +.. autoclass:: OperatorClassDict + +.. automethod:: OperatorClassDict.from_map diff --git a/docs/operfamily.rst b/docs/operfamily.rst new file mode 100644 index 0000000..b2d407d --- /dev/null +++ b/docs/operfamily.rst @@ -0,0 +1,35 @@ +Operator Families +================= + +.. module:: pyrseas.dbobject.operfamily + +The :mod:`operfamily` module defines two classes: class +:class:`OperatorFamily` derived from :class:`DbSchemaObject` and class +:class:`OperatorFamilyDict` derived from :class:`DbObjectDict`. + +Operator Family +--------------- + +:class:`OperatorFamily` is derived from :class:`DbSchemaObject` and +represents a `Postgres operator family +`_, +a grouping of related operator classes. + +.. autoclass:: OperatorFamily + +.. automethod:: OperatorFamily.extern_key + +.. automethod:: OperatorFamily.identifier + +.. automethod:: OperatorFamily.create + +Operator Family Dictionary +-------------------------- + +:class:`OperatorFamilyDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of operator families in a database. + +.. autoclass:: OperatorFamilyDict + +.. automethod:: OperatorFamilyDict.from_map diff --git a/docs/overview.rst b/docs/overview.rst new file mode 100644 index 0000000..ca1b085 --- /dev/null +++ b/docs/overview.rst @@ -0,0 +1,117 @@ +.. -*- coding: utf-8 -*- + +Overview +======== + +Pyrseas provides utilities to maintain a `PostgreSQL +`_ database schema. Its purpose is to +enhance and follow through on the concepts of the `Andromeda Project +`_. + +Whereas Andromeda expects the database designer or developer to +provide a single `YAML `_ specification file of the +database to be created, Pyrseas allows the development database to be +created using the familiar SQL CREATE statements. The developer can +then run the `dbtoyaml` utility to generate the YAML specification +from the database. The spec can then be stored in any desired version +control (VCS) repository. Similarly, she can add columns or modify +tables or other objects using SQL ALTER statements and regenerate the +YAML spec with dbtoyaml. + +When ready to create or upgrade a test or production database, the +`yamltodb` utility can be used with the YAML spec as input, to generate +a script of SQL CREATE or ALTER statements to modify the database so +that it matches the input spec. + +A third tool, `dbaugment`, can be used to add custom attributes and +supporting objects to a given schema. For example, an `updated` +column can be added to various tables, together with trigger functions +to ensure the columns are automatically modified as changes are made. + +Use Cases +--------- + +The following sections discuss the main scenarios where Pyrseas +tools may be helpful. + +Version Control +--------------- + +The case for implementing a tool to facilitate version control over +SQL databases was made in a couple of blog posts: `Version +Control, Part 1: Pre-SQL +`_ +and `Version Control, Part 2: SQL Databases +`_. In +summary, SQL data definition commands are generally incompatible with +traditional version control approaches which usually require +comparisons (diffs) between revisions of source files. + +A refinement of the approach described in the aforementioned blog +posts will be of interest to users with many objects in their database +schemas, i.e., many tables, views, functions, and other more complex +objects. Instead of storing a complete database specification in a +single YAML file, by using the `--multiple-files` option to +:program:`dbtoyaml`, the specification can be broken down into files +corresponding, generally, to a single database object. This allows a +VCS **diff** facility to easily highlight database changes. Please +refer to the :doc:`dbtoyaml` and :doc:`yamltodb` utilities for further +details. + +The Pyrseas version control tools are not designed to be the ultimate +SQL database version control solution. Instead, they are aimed at +assisting two or more developers or DBAs in sharing changes to the +underlying database as they implement a database application. The +sharing can occur through a distributed or centralized VCS. The +Pyrseas tools may even be used by a single DBA in conjunction with a +distributed VCS to quickly explore alternative designs. The tools can +also help to share changes with a conventional QA team, but may +require additional controls for final releases and production +installations. + +Supplementary Actions +--------------------- + +In many instances, a database schema needs to be supplemented by +rarely-modified data kept in certain tables, e.g., a +codes-descriptions table. The data import and export features, +controlled by `datacopy` configuration parameter (see +:doc:`configitems` for details) facilitates this need. + +In other cases, DBAs may want to standardize certain additional table +columns or processing. For example, they may want to capture the user +and time of modification of a certain set of tables using a common +procedure. The :doc:`dbaugment` utility was introduced to support +these needs. + +Generating SQL by determining what changed between one schema version +and another is sometimes not sufficient. Although the change may be +as simple as adding a column to a table and adding a referential +constraint to the new column, if the tables already have data, it may +not be possible to run the SQL generated by :program:`yamltodb`. This +requires manual intervention by the DBAs or developers. The project +would like to assist with these types of changes. The blog post `The +Future of Pyrseas: Part 1 +`_ +is a first step in discussing this requirement. + +Naming +------ + +The project name comes from `Python `_, the +programming language, and `Perseas +`_ [#]_, the Greek mythological +hero who rescued Andromeda from a sea monster [#]_. It is hoped that +Pyrseas will rescue the Andromeda project . You can pronounce +Pyrseas like the hero. + + +.. rubric:: Footnotes + +.. [#] The common English name for Perseas is Perseus and the Ancient + Greek name is Perseos. However, in modern Greek Περσέας_ is the + more common spelling for the mythical hero. + +.. _Περσέας: https://en.wiktionary.org/wiki/%CE%A0%CE%B5%CF%81%CF%83%CE%AD%CE%B1%CF%82 + +.. [#] He is better known for having killed Medusa. diff --git a/docs/predefaug.rst b/docs/predefaug.rst new file mode 100644 index 0000000..1dc3456 --- /dev/null +++ b/docs/predefaug.rst @@ -0,0 +1,93 @@ +.. _predef-aug: + +Predefined Database Augmentations +================================= + +These augmentations are specified in the ``config.yaml`` configuration +file distributed with Pyrseas' :program:`dbaugment`. + +Columns +------- + +These are predefined column specifications that can be added to +tables, e.g., in various audit column combinations (see `Audit +Columns`_ below). + +- created_by_ip_address: An INET column to record the IP address which + originated the current row. + +- created_by_user: A VARCHAR(63) column to record the user, e.g., + CURRENT_USER, who created the current row. + +- created_date: A DATE column that defaults to CURRENT_DATE. + +- created_timestamp: A TIMESTAMP WITH TIME ZONE column to record the + date and time when the current row was created. + +- modified_by_ip_address: An INET column to record the IP address + which originated the last modification to the current row. + +- modified_by_user: A VARCHAR(63) column to record the user, e.g., + CURRENT_USER, who last modified the current row. + +- modified_timestamp: A TIMESTAMP WITH TIME ZONE column to record the + date and time when the current row was last modified. + +Functions +--------- + +The following are predefined trigger functions which are used to +implement various augmentations. The source for each function, +written in PL/pgSQL, is specified in a function template, named with a +``functempl_`` prefixed to the function name. + +- Audit when modified (``audit_modified``): This function provides the + CURRENT_TIMESTAMP value for audit columns. + +- Default audit (``audit_default``): This function provides the + CURRENT_USER and CURRENT_TIMESTAMP for audit columns. + +- Full audit (``audit_full``): For SQL INSERTs, this function provides + values for the user who created the row, the CURRENT_TIMESTAMP and + the IP address for both the ``created_`` and ``modified_`` audit + columns. For UPDATEs, it retains the existing values in the + ``created_`` columns and supplies current values for the + ``modified_`` columns. + +In addition, the following helper functions are defined in schema +``pyrseas``: + +- get_session_variable +- set_session_variable + +A variant of ``get_session_variable`` is invoked by the ``audit_full`` +function to retrieve the actual (logged-on) user and IP address. In +web applications, the user that connects to the database is typically +the system user running the web server, rather than the application +(logged on) user. The application can invoke the +``pyrseas.set_session_variable`` function to supply the application +user and IP address so that the audit trail will reflect the +application context corrrectly. + +Audit Columns +------------- + +These are predefined combinations of columns to be added to tables to +record audit trail information. They may also include triggers to be +invoked to maintain the column values. + +- created_date_only: This is the simplest audit trail that adds a + ``created_date`` column which defaults to the CURRENT_DATE. + +- modified_only: This is another simple audit trail. It adds a + ``modified_timestamp`` column which is supplied by a trigger named + `table_name`\_20_audit_modified_only. + +- default: This is the default for audit columns. It adds the columns + ``modified_by_user`` and ``modified_timestamp`` and a trigger named + `table_name`\_20_audit_default to fill in the columns. + +- full: This is the most extensive audit trail combination. It adds + ``created_`` and ``modified_`` columns for user, IP address and + timestamp. It also adds a trigger named + `table_name`\_20_audit_full. diff --git a/docs/rule.rst b/docs/rule.rst new file mode 100644 index 0000000..aff3dd3 --- /dev/null +++ b/docs/rule.rst @@ -0,0 +1,35 @@ +Rules +===== + +.. module:: pyrseas.dbobject.rule + +The :mod:`rule` module defines two classes, :class:`Rule` and +:class:`RuleDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Rule +---- + +:class:`Rule` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a `Postgres +rewrite rule +`_. + +.. autoclass:: Rule + +.. automethod:: Rule.identifier + +.. automethod:: Rule.to_map + +.. automethod:: Rule.create + +Rule Dictionary +--------------- + +:class:`RuleDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of rewrite rules in a database. + +.. autoclass:: RuleDict + +.. automethod:: RuleDict.from_map diff --git a/docs/schema.rst b/docs/schema.rst new file mode 100644 index 0000000..f99d26b --- /dev/null +++ b/docs/schema.rst @@ -0,0 +1,57 @@ +Schemas +======= + +.. module:: pyrseas.dbobject.schema + +The :mod:`schema` module defines two classes, :class:`Schema` and +:class:`SchemaDict`, derived from :class:`DbObject` and +:class:`DbObjectDict`, respectively. + +Schema +------ + +:class:`Schema` is derived from :class:`~pyrseas.dbobject.DbObject` +and represents a database schema or Postgres namespace, i.e., a +collection of tables and other objects. The 'public' schema is +currently treated specially as in most contexts an unqualified object +is assumed to be part of it, e.g., table "t" is usually shorthand for +table "public.t." The 'pyrseas' schema, if present, is excluded as it +is only intended for use by :program:`dbaugment` or other Pyrseas +internal purposes. + +.. autoclass:: Schema + +.. automethod:: Schema.extern_dir + +.. automethod:: Schema.to_map + +.. automethod:: Schema.create + +.. automethod:: Schema.drop + +.. automethod:: Schema.data_import + +Schema Dictionary +----------------- + +:class:`SchemaDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of schemas in a database. Certain internal +schemas (information_schema, pg_catalog, etc.) owned by the 'postgres' +user are excluded. + +.. autoclass:: SchemaDict + +Method :meth:`from_map` is called from :class:`Database` +:meth:`~pyrseas.database.Database.from_map` to start a recursive +interpretation of the input map. The :obj:`inmap` argument is the same +as input to the :meth:`~pyrseas.database.Database.diff_map` method of +:class:`Database`. The :obj:`newdb` argument is the holder of +:class:`~pyrseas.dbobject.DbObjectDict`-derived dictionaries which is +filled in as the recursive interpretation proceeds. + +.. automethod:: SchemaDict.from_map + +.. automethod:: SchemaDict.to_map + +.. automethod:: SchemaDict.data_import diff --git a/docs/table.rst b/docs/table.rst new file mode 100644 index 0000000..fc7cd37 --- /dev/null +++ b/docs/table.rst @@ -0,0 +1,178 @@ +Tables, Views and Sequences +=========================== + +.. module:: pyrseas.dbobject.table + +The :mod:`table` and :mod:`view` modules define six classes, +:class:`DbClass` derived from :class:`DbSchemaObject`, classes +:class:`Sequence`, :class:`Table` and :class:`View` derived from +:class:`DbClass`, :class:`MaterializedView` derived from +:class:`View`, and :class:`ClassDict`, derived from +:class:`DbObjectDict`. + +Database Class +-------------- + +Class :class:`DbClass` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a table, view +or sequence as defined in the Postgres `pg_class` catalog. + +.. autoclass:: DbClass + +Sequence +-------- + +Class :class:`Sequence` is derived from :class:`DbClass` and +represents a `sequence generator +`_. +Its :attr:`keylist` attributes are the schema name and the sequence +name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has the following structure:: + + {'sequence seq1': + {'cache_value': 1, + 'data_type': 'integer', + 'increment_by': 1, + 'max_value': None, + 'min_value': None, + 'owner_column': 'c1', + 'owner_table': 't1', + 'start_value': 1 + } + } + +.. autoclass:: Sequence + +.. automethod:: Sequence.get_attrs + +.. automethod:: Sequence.get_dependent_table + +.. automethod:: Sequence.to_map + +.. automethod:: Sequence.create + +.. automethod:: Sequence.add_owner + +.. automethod:: Sequence.alter + +.. automethod:: Sequence.drop + +Table +----- + +Class :class:`Table` is derived from :class:`DbClass` and represents a +database table. Its :attr:`keylist` attributes are the schema name and +the table name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has a structure similar to the following:: + + {'table t1': + {'columns': + [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'smallint'}}, + {'c4': {'type': 'date', 'default': 'now()'}} + ], + 'description': "this is the comment for table t1", + 'primary_key': + {'t1_prim_key': + {'columns': ['c1', 'c2']} + }, + 'foreign_keys': + {'t1_fgn_key1': + {'columns': ['c2', 'c3'], + 'references': + {'table': 't2', 'columns': ['pc2', 'pc1']} + }, + 't1_fgn_key2': + {'columns': ['c2'], + 'references': {'table': 't3', 'columns': ['qc1']} + } + }, + 'unique_constraints': {...}, + 'indexes': {...} + } + } + +The values for :obj:`unique_constraints` and :obj:`indexes` follow a +pattern similar to :obj:`primary_key`, but there can be more than one +such specification. + +.. autoclass:: Table + +.. automethod:: Table.column_names + +.. automethod:: Table.to_map + +.. automethod:: Table.create + +.. automethod:: Table.drop + +.. automethod:: Table.diff_options + +.. automethod:: Table.alter + +.. automethod:: Table.alter_drop_columns + +.. automethod:: Table.data_export + +.. automethod:: Table.data_import + +Class Dictionary +---------------- + +Class :class:`ClassDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict` and represents the collection +of tables, views and sequences in a database. + +.. autoclass:: ClassDict + +.. automethod:: ClassDict.from_map + + +.. module:: pyrseas.dbobject.view + +View +---- + +Class :class:`View` is derived from :class:`DbClass` and represents a +database view. Its :attr:`keylist` attributes are the schema name and +the view name. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`ClassDict.from_map` has a structure similar to the following:: + + {'view v1': + {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'date'}}], + 'definition': " SELECT ...;", + 'description': "this is the comment for view v1" + } + } + + +.. autoclass:: View + +.. automethod:: View.to_map + +.. automethod:: View.create + +.. automethod:: View.alter + +Materialized View +----------------- + +Class :class:`MaterializedView` is derived from :class:`View` and +represents a `materialized view +`_. Its +:attr:`keylist` attributes are the schema name and the view name. + +.. autoclass:: MaterializedView + +.. automethod:: MaterializedView.to_map + +.. automethod:: MaterializedView.create diff --git a/docs/testing.rst b/docs/testing.rst new file mode 100644 index 0000000..5b56903 --- /dev/null +++ b/docs/testing.rst @@ -0,0 +1,175 @@ +.. _testing: + +Testing +======= + +The majority of Pyrseas' capabilities are exercised and verified via +unit tests written using `pytest +`_. The tests can be run from the +command line by most users, e.g., + +:: + + pytest-3 tests/dbobject/test_table.py + pytest-3 tests/dbobject/test_trigger.py -k test_create_trigger + pytest-3 tests/functional + +The first ``pytest-3`` command above runs all tests related to tables, +mapping, creating, dropping, etc. The second one executes a single +test to generate SQL to create a trigger. The third runs all the +functional tests. Please review the `pytest documentation +`_ for further options. + +Environment Variables +--------------------- + +By default, the tests use a Postgres database named ``pyrseas_testdb`` +which is created if it doesn't already exist. The tests are run as the +logged in user, using the ``USER`` Unix/Linux environment variable (or +``USERNAME`` under Windows). They access Postgres on the local host +using the default port number (5432). + +The following four environment variables can be used to change the +defaults described above: + + - PYRSEAS_TEST_DB + - PYRSEAS_TEST_USER + - PYRSEAS_TEST_HOST + - PYRSEAS_TEST_PORT + +Restrictions +------------ + +Unless the test database exists and the user running the tests has +access to it, the user role will need CREATEDB privilege. + +Most tests do not require special privileges. However, certain tests +may require Postgres SUPERUSER privilege. Such tests will normally be +skipped if the user lacks the privilege. + +Most tests do not require installation of supporting Postgres +packages. However, a few tests rely on the availability of Postgres +``contrib`` modules such as the `spi module +`_ or +procedural languages such as ``plperl``, ``plpython3u`` or ``plr``. + +On Windows, it is necessary to install Perl in order to run some of +the tests (most Linux or Unix variants already include it as part of +their normal distribution). The last time we checked, a suitable +choice appeared to be Strawberry Perl which can be downloaded from +http://strawberryperl.com/releases.html. However, the default +installation is placed in ``C:\strawberry`` and can hold a single Perl +version. Furthermore, some Postgres versions may be linked with +non-current Perl versions. It is recommended that the latest Perl +version be installed as this will usually give the fewest test +failures. See `this blog post +`_ +for more details. + +The COLLATION tests require the +``fr_FR.utf8`` locale (or ``French.France.1252`` language on Windows) +to be installed. + +Testing Checklist +----------------- + +The following is a summary list of steps needed to test Pyrseas on a +new machine. Refer to :ref:`development` for details on how to +accomplish a given installation task. "Package manager" refers to the +platform's package management system utility such as ``apt-get`` or +``yum``. Installation from PyPI can be done with ``pip``. Some +operations require administrative or superuser privileges, at either +the operating system or Postgres level. + + - Install Git using package manager or from + https://git-scm.com/download (on Windows, prefer Git Bash) + + - ``git clone git://github.com/perseas/Pyrseas.git`` + + - Install Python 3.7 or higher, using package manager or from + installers at https://www.python.org/downloads/. + + - Install Postgres 13, 12, 11 or 10, using package manager or + binary installers at https://www.postgresql.org/download/ + + .. note:: On Linux, make sure you install the contrib and plperl + packages, e.g., on Debian, postgresql-contrib-n and + postgresql-plperl-n (where `n` is the Postgres + version number) + + - Install Psycopg, using package manager, or from PyPI + (https://pypi.org/project/psycopg/). + + .. note:: On Windows, you may first want to install a version of + Microsoft Visual Studio from `here`_. An alternative + that may work is `MinGW `_. See + `these blog`_ `posts`_ for more details. + + .. _here: https://www.microsoft.com/en-us/download/developer-tools.aspx + + .. _these blog: https://pyrseas.wordpress.com/2012/09/25/testing-python-and-postgresql-on-windows-part-2/ + + .. _posts: https://pyrseas.wordpress.com/2012/09/28/testing-python-and-postgresql-on-windows-part-3/ + + - Install PyYAML, using package manager, or from PyPI + (https://pypi.org/project/PyYAML/) or + http://pyyaml.org/download/pyyaml/. + + - Install pytest, using package manager, or from PyPI + (https://pypi.org/project/pytest/). + + - Install Tox, using package manager, or from PyPI + (https://pypi.org/project/tox/) + + - On Windows, install Perl (see discussion above under + "Restrictions"). On Linux, usually Perl is already available. + + - As **postgres** user, using psql or pgAdmin, create a test user, + e.g., your name. The user running tests must have at a minimum + createdb privilege, in order to create the test database. To run + *all* the tests, the user also needs superuser privilege. + + - Create a Postgres password file, e.g., on Linux: ``~/.pgpass``, on + Windows: ``%APPDATA%\postgresql\pgpass.conf``. + + - Create directories to hold tablespaces, e.g., ``/extra/pg/13.0/ts1`` + on Linux, ``C:\\extra\\pg\\13.0\\ts1`` on Windows. The directories + need to be owned by the **postgres** user. This may be tricky on + older Windows versions, but the command ``cacls /E /G + postgres:F`` should suffice. Using ``psql``, create tablespaces + **ts1** and **ts2**, e.g., ``CREATE TABLESPACE ts1 LOCATION + ''`` (on Windows, you'll have to use, e.g., + ``E'C:\\dir\\ts1'``, to specify the directory). + + - Install the locale ``fr_FR.utf8`` on Linux/Unix or the language + ``French.France.1252`` on Windows. + + - On Debian and derivatives, this can be done with the command:: + + sudo dpkg-reconfigure locales + + - On Windows, open the Control Panel, select Date, Time, Language, + and Regional Options, then Regional and Language Options (or Add + other languages), click on the Advanced tab in the dialog and + then choose “French (France)” from the dropdown. Finally, click + OK and respond to any subsequent prompts to install the locale, + including rebooting the machine. + + - Change to the Pyrseas source directory (created by the second step above). + + - Define the ``PYTHONPATH`` environment variable to the Pyrseas source + directory, e.g., on Linux, ``export PYTHONPATH=$PWD``, on + Windows, ``set PYTHONPATH=%USERPROFILE%\somedir\Pyrseas``. + + - Define the ``PG1N0_PORT`` environment variables (where ``1N`` + represents the major Posgres version, e.g., 15, 11) to point to the + corresponding Postgres connection ports. + + - Invoke ``tox``. This will create virtualenvs in a ``.tox`` + subdirectory, + install Pyrseas and its prerequisites (Psycopg and PyYAML) into + each virtualenv and run the unit tests for each combination of + Postgres and Python. + +If you find any problems with the instructions above, please open an +issue on `GitHub `_. diff --git a/docs/textsearch.rst b/docs/textsearch.rst new file mode 100644 index 0000000..c03c45d --- /dev/null +++ b/docs/textsearch.rst @@ -0,0 +1,102 @@ +Text Search Objects +=================== + +.. module:: pyrseas.dbobject.textsearch + +The :mod:`textsearch` module defines eight classes: classes +:class:`TSConfiguration`, :class:`TSDictionary`, :class:`TSParser` and +:class:`TSTemplate` derived from :class:`DbSchemaObject`, and classes +:class:`TSConfigurationDict`, :class:`TSDictionaryDict`, +:class:`TSParserDict` and :class:`TSTemplateDict` derived from +:class:`DbObjectDict`. + +Text Search Configuration +------------------------- + +:class:`TSConfiguration` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search configuration +`_. + +.. autoclass:: TSConfiguration + +.. automethod:: TSConfiguration.to_map + +.. automethod:: TSConfiguration.create + +Text Search Configuration Dictionary +------------------------------------ + +:class:`TSConfigurationDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search configurations in a database. + +.. autoclass:: TSConfigurationDict + +.. automethod:: TSConfigurationDict.from_map + +Text Search Dictionary +---------------------- + +:class:`TSDictionary` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search dictionary +`_. + +.. autoclass:: TSDictionary + +.. automethod:: TSDictionary.create + +Text Search Dictionary Dictionary +--------------------------------- + +:class:`TSDictionaryDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a Python dictionary +that represents the collection of text search dictionaries in a +database. + +.. autoclass:: TSDictionaryDict + +.. automethod:: TSDictionaryDict.from_map + +Text Search Parser +------------------ + +:class:`TSParser` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search parser +`_. + +.. autoclass:: TSParser + +.. automethod:: TSParser.create + +Text Search Parser Dictionary +----------------------------- + +:class:`TSParserDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search parsers in a database. + +.. autoclass:: TSParserDict + +.. automethod:: TSParserDict.from_map + +Text Search Template +-------------------- + +:class:`TSTemplate` is derived from :class:`DbSchemaObject` and +represents a `Postgres text search template +`_. + +.. autoclass:: TSTemplate + +.. automethod:: TSTemplate.create + +Text Search Template Dictionary +------------------------------- + +:class:`TSTemplateDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of text search templates in a database. + +.. autoclass:: TSTemplateDict + +.. automethod:: TSTemplateDict.from_map diff --git a/docs/trigger.rst b/docs/trigger.rst new file mode 100644 index 0000000..3d5e097 --- /dev/null +++ b/docs/trigger.rst @@ -0,0 +1,35 @@ +Triggers +======== + +.. module:: pyrseas.dbobject.trigger + +The :mod:`trigger` module defines two classes, :class:`Trigger` and +:class:`TriggerDict`, derived from :class:`DbSchemaObject` and +:class:`DbObjectDict`, respectively. + +Trigger +------- + +:class:`Trigger` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a Postgres +`regular or constraint trigger +`_. + +.. autoclass:: Trigger + +.. automethod:: Trigger.identifier + +.. automethod:: Trigger.to_map + +.. automethod:: Trigger.create + +Trigger Dictionary +------------------ + +:class:`TriggerDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of triggers in a database. + +.. autoclass:: TriggerDict + +.. automethod:: TriggerDict.from_map diff --git a/docs/type.rst b/docs/type.rst new file mode 100644 index 0000000..8482f9e --- /dev/null +++ b/docs/type.rst @@ -0,0 +1,120 @@ +Types and Domains +================= + +.. module:: pyrseas.dbobject.dbtype + +The :mod:`dbtype` module defines seven classes, :class:`DbType` +derived from :class:`DbSchemaObject`, :class:`BaseType`, +:class:`Composite`, :class:`Enum`, :class:`Domain` and :class:`Range` +derived from :class:`DbType`, and :class:`TypeDict` derived from and +:class:`DbObjectDict`. + +Database Type +------------- + +Class :class:`DbType` is derived from +:class:`~pyrseas.dbobject.DbSchemaObject` and represents a SQL type or +domain as defined in the Postgres `pg_type` catalog. + +.. autoclass:: DbType + +Base Type +--------- + +:class:`BaseType` is derived from :class:`~pyrseas.dbobject.DbType` +and represents a Postgres `user-defined base type +`_. + +The map returned by :meth:`to_map` and expected as argument by +:meth:`TypeDict.from_map` has the following structure (not all fields +need be present):: + + {'type t1': + {'alignment': 'double', + 'analyze': 'analyze_func', + 'category': 'U', + 'delimiter': ',', + 'input': 'input_func', + 'internallength': 'variable', + 'output': 'output_func', + 'preferred': 'true', + 'receive': 'receive_func', + 'send': 'send_func', + 'storage': 'plain' + 'typmod_in': 'typmod_in_func', + 'typmod_out': 'typmod_out_func' + } + } + +.. autoclass:: BaseType + +.. automethod:: BaseType.to_map + +.. automethod:: BaseType.create + +.. automethod:: BaseType.drop + +Composite +--------- + +:class:`Composite` is derived from :class:`~pyrseas.dbobject.DbType` +and represents a standalone `composite type +`_. + +.. autoclass:: Composite + +.. automethod:: Composite.to_map + +.. automethod:: Composite.create + +.. automethod:: Composite.alter + +Enum +---- + +:class:`Enum` is derived from :class:`~pyrseas.dbobject.DbType` and +represents an `enumerated type +`_. + +.. autoclass:: Enum + +.. automethod:: Enum.create + +Domain +------ + +:class:`Domain` is derived from :class:`~pyrseas.dbobject.DbType` and +represents a `domain +`_. + +.. autoclass:: Domain + +.. automethod:: Domain.to_map + +.. automethod:: Domain.create + +Range +----- + +:class:`Range` is derived from :class:`~pyrseas.dbobject.DbType` and +represents a `Postgres range type +`_. + +.. autoclass:: Range + +.. automethod:: Range.to_map + +.. automethod:: Range.create + +.. automethod:: Range.alter + +Type Dictionary +--------------- + +:class:`TypeDict` is derived from +:class:`~pyrseas.dbobject.DbObjectDict`. It is a dictionary that +represents the collection of domains and enums in a database. + +.. autoclass:: TypeDict + +.. automethod:: TypeDict.from_map diff --git a/docs/yamltodb.rst b/docs/yamltodb.rst new file mode 100644 index 0000000..fa242e4 --- /dev/null +++ b/docs/yamltodb.rst @@ -0,0 +1,129 @@ +yamltodb - YAML to Database +=========================== + +Name +---- + +yamltodb -- generate SQL statements to update a Postgres database to +match the schema specified in a YAML file + +Synopsis +-------- + +:: + + yamltodb [option...] dbname [spec] + +Description +----------- + +:program:`yamltodb` is a utility for generating SQL statements to +update a Postgres database so that it will match the schema +specified in an input `YAML `_ formatted +specification file. + +For example, given the input file shown under :doc:`dbtoyaml`, +:program:`yamltodb`, when run against a newly-created database, +outputs the following SQL statements:: + + CREATE SCHEMA s1; + ALTER SCHEMA s1 OWNER TO bob; + GRANT ALL ON SCHEMA s1 TO bob; + GRANT ALL ON SCHEMA s1 TO alice; + CREATE TABLE t1 ( + c1 integer NOT NULL, + c2 smallint, + c3 boolean DEFAULT false, + c4 text); + ALTER TABLE t1 OWNER TO alice; + CREATE TABLE s1.t2 ( + c21 integer NOT NULL, + c22 character varying(16)); + ALTER TABLE s1.t2 OWNER TO bob; + GRANT ALL ON TABLE s1.t2 TO bob; + GRANT SELECT ON TABLE s1.t2 TO PUBLIC; + GRANT INSERT, DELETE, UPDATE ON TABLE s1.t2 TO alice WITH GRANT OPTION; + GRANT INSERT ON TABLE s1.t2 TO carol; + ALTER TABLE t1 ADD CONSTRAINT t1_c2_check CHECK (c2 > 123); + ALTER TABLE t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1); + ALTER TABLE s1.t2 ADD CONSTRAINT t2_pkey PRIMARY KEY (c21); + ALTER TABLE t1 ADD CONSTRAINT t1_c2_fkey FOREIGN KEY (c2) REFERENCES s1.t2 (c21); + +Options +------- + +:program:`yamltodb` accepts the following command-line arguments (in +addition to the :doc:`cmdargs`): + +.. program:: yamltodb + +**dbname** + + Specifies the name of the database whose schema is to analyzed. + +**spec** + + Specifies the location of the YAML specification. If this is + omitted or specified as a single or double dash, the specification + is read from the program's standard input. However, if the + :option:`--multiple-files` option is used, that takes precedence. + +.. cmdoption:: -m, --multiple-files + + Specifies that input should be taken from YAML specification files + present in a two-level (metadata) directory tree. See `Multiple + File Output` under :doc:`dbtoyaml` for further details. + +.. cmdoption:: -n + --schema + + Compare only a schema matching `schema`. By default, all schemas + are compared. Multiple schemas can be compared by using multiple + :option:`-n` switches. + +.. cmdoption:: -1 + --single-transaction + + Wrap the generated statements in BEGIN/COMMIT. This ensures that + either all the statements complete successfully, or no changes are + applied. + +.. cmdoption:: -u, --update + + Execute the generated statements against the database mentioned in + **dbname**. This implies the :option:`--single-transaction` + option. + +.. cmdoption:: --revert + + Generate SQL in reversion mode, that is, to undo the changes that + would normally be generated. For example, if without this option, + the SQL would be a ``DROP TABLE``, the :option:`--revert` option + generates a ``CREATE TABLE`` with all the columns, constraints and + other objects associated with the table being dropped. + + This option is experimental and currently has only been + implemented for schemas and sequences. + +Examples +-------- + +Given a YAML file named ``moviesdb.yaml``, to generate SQL statements +to update a database called `mymovies`:: + + yamltodb mymovies moviesdb.yaml + +To generate the statements as above and immediately update `mymovies`:: + + yamltodb mymovies moviesdb.yaml | psql mymovies + +or:: + + yamltodb --update mymovies moviesdb.yaml + +To generate the statements directly from the output of +:program:`dbtoyaml` (against a different database), with statements +enclosed in a single transaction, and save the statements in a file +named ``mymovies.sql``:: + + dbtoyaml devmovies | yamltodb -1 mymovies -o mymovies.sql diff --git a/pyrseas/.vscode/launch.json b/pyrseas/.vscode/launch.json new file mode 100644 index 0000000..4adb9d5 --- /dev/null +++ b/pyrseas/.vscode/launch.json @@ -0,0 +1,12 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + {"args": [ + "-H", "192.168.128.7", "-U", "postgres", "test_db", "C:\\Projects\\new_dbsync\\db\\ddl\\test_db_schema.yaml" + ] + } + ] +} \ No newline at end of file diff --git a/pyrseas/__init__.py b/pyrseas/__init__.py new file mode 100644 index 0000000..9d1bb72 --- /dev/null +++ b/pyrseas/__init__.py @@ -0,0 +1 @@ +__version__ = '0.10.0' diff --git a/pyrseas/augment/__init__.py b/pyrseas/augment/__init__.py new file mode 100644 index 0000000..a2413c9 --- /dev/null +++ b/pyrseas/augment/__init__.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment + ~~~~~~~~~~~~~~~ + + This defines two low level classes. Most Database Augmenter + classes are derived from either DbAugment or DbAugmentDict. +""" + + +class DbAugment(object): + "A database augmentation object" + + keylist = ['name'] + """List of attributes that uniquely identify the object""" + + def __init__(self, **attrs): + """Initialize the augmentation object from a dictionary of attributes + + :param attrs: the dictionary of attributes + """ + for key, val in list(attrs.items()): + setattr(self, key, val) + + +class DbAugmentDict(dict): + """A dictionary of database augmentations, all of the same type""" + + cls = DbAugment + """The class, derived from :class:`DbAugment` that the objects belong to. + """ diff --git a/pyrseas/augment/audit.py b/pyrseas/augment/audit.py new file mode 100644 index 0000000..b1146e1 --- /dev/null +++ b/pyrseas/augment/audit.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.audit + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgAuditColumn derived from + DbAugment and CfgAuditColumnDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugment, DbAugmentDict +from pyrseas.dbobject import split_schema_obj + + +class CfgAuditColumn(DbAugment): + """An augmentation that adds automatically maintained audit columns""" + + keylist = ['name'] + + def apply(self, table, augdb): + """Apply audit columns to argument table. + + :param table: table to which columns/triggers will be added + :param augdb: augment dictionaries + """ + currdb = augdb.current + sch = table.schema + for col in self.columns: + augdb.columns[col].apply(table) + if hasattr(self, 'triggers'): + for trg in self.triggers: + augdb.triggers[trg].apply(table) + for newtrg in table.triggers: + fncsig = table.triggers[newtrg].procedure + (sch, fnc) = split_schema_obj(fncsig, table.schema) + if (sch, fncsig) not in currdb.functions: + newfunc = augdb.functions[fnc].apply( + sch, augdb.columns.col_trans_tbl, augdb) + # add new function to the current db + augdb.add_func(sch, newfunc) + augdb.add_lang(newfunc.language) + + +class CfgAuditColumnDict(DbAugmentDict): + "The collection of audit column augmentations" + + cls = CfgAuditColumn + + def __init__(self, config): + for aud in config: + self[aud] = CfgAuditColumn(name=aud, **config[aud]) + + def from_map(self, inaudcols): + """Initialize the dictionary of functions by converting the input map + + :param inaudcols: YAML map defining the audit column configuration + """ + for aud in inaudcols: + audcol = CfgAuditColumn(name=aud) + for attr in inaudcols[aud]: + if attr == 'columns': + audcol.columns = [col for col in inaudcols[aud][attr]] + elif attr == 'triggers': + audcol.triggers = [col for col in inaudcols[aud][attr]] + self[audcol.name] = audcol diff --git a/pyrseas/augment/column.py b/pyrseas/augment/column.py new file mode 100644 index 0000000..ee917c2 --- /dev/null +++ b/pyrseas/augment/column.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.column + ~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgColumn derived from + DbAugment and CfgColumnDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject.column import Column + + +class CfgColumn(DbAugment): + "A configuration column definition" + + keylist = ['name'] + + def apply(self, table): + """Add columns to the table passed in. + + :param table: table to which the columns will be added + """ + if self.name in table.column_names(): + for col in table.columns: + if col.name == self.name: + col.type = self.type + if hasattr(self, 'not_null'): + col.not_null = self.not_null + if hasattr(self, 'default'): + col.default = self.default + else: + dct = self.__dict__.copy() + dct.pop('name') + dct.pop('type') + newcol = Column(self.name, table.schema, table.name, 0, self.type, + **dct) + newcol._table = table + table.columns.append(newcol) + + +class CfgColumnDict(DbAugmentDict): + "The collection of configuration columns" + + cls = CfgColumn + + def __init__(self, config): + self.col_trans_tbl = [] + for col in config: + if not 'name' in config[col]: + config[col]['name'] = col + self[col] = CfgColumn(**config[col]) + self.col_trans_tbl.append(('{{%s}}' % col, self[col].name)) + + def from_map(self, incols): + """Initialize the dictionary of columns by converting the input dict + + :param incols: YAML dictionary defining the columns + """ + renames = False + for col in incols: + if col in self: + ccol = self[col] + else: + self[col] = ccol = CfgColumn(name=col) + for attr, val in list(incols[col].items()): + setattr(ccol, attr, val) + if attr == 'name': + renames = True + if renames: + self.col_trans_tbl = [('{{%s}}' % col, self[col].name) + for col in self] diff --git a/pyrseas/augment/function.py b/pyrseas/augment/function.py new file mode 100644 index 0000000..bbdb60f --- /dev/null +++ b/pyrseas/augment/function.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.function + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgFunction derived from + DbAugment and CfgFunctionDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject.function import Function + + +class CfgFunctionSource(DbAugment): + "A configuration function source or part thereof" + pass + + +class CfgFunctionTemplate(CfgFunctionSource): + "A configuration function source template" + + pass + + +class CfgFunctionSourceDict(DbAugmentDict): + + cls = CfgFunctionSource + + def __init__(self, cfg_templates): + for templ in cfg_templates: + src = cfg_templates[templ] + dct = {'source': src} + self[templ] = CfgFunctionTemplate(name=templ, **dct) + + def from_map(self, intempls): + """Initialize the dict of templates by converting the input list + + :param intempls: YAML list defining the function templates + """ + for templ in intempls: + self[templ] = CfgFunctionTemplate( + name=templ, source=intempls[templ]) + + +class CfgFunction(DbAugment): + "A configuration function definition" + + keylist = ['name', 'arguments'] + + def apply(self, schema, trans_tbl, augdb): + """Add a function to a given schema. + + :param schema: name of the schema in which to create the function + :param trans_tbl: translation table + :param augdb: augmenter dictionaries + """ + newdict = self.__dict__.copy() + newdict.pop('name') + newdict.pop('description') + newfunc = Function(self.name, schema, self.description, None, [], + **newdict) + src = newfunc.source + if '{{' in src and '}}' in src: + pref = src.find('{{') + prefix = src[:pref] + suf = src.find('}}') + suffix = src[suf + 2:] + tmplkey = src[pref + 2:suf] + if tmplkey not in augdb.funcsrcs: + if '{{'+tmplkey+'}}' not in [pat for (pat, repl) in trans_tbl]: + raise KeyError("Function template '%s' not found" % + tmplkey) + else: + newfunc.source = prefix + augdb.funcsrcs[tmplkey].source + \ + suffix + + for (pat, repl) in trans_tbl: + if '{{' in newfunc.source: + newfunc.source = newfunc.source.replace(pat, repl) + if '{{' in newfunc.name: + newfunc.name = newfunc.name.replace(pat, repl) + if '{{' in newfunc.description: + newfunc.description = newfunc.description.replace(pat, repl) + return newfunc + + +class CfgFunctionDict(DbAugmentDict): + "The collection of configuration functions" + + cls = CfgFunction + + def __init__(self, config): + for func in config: + fncdict = config[func] + paren = func.find('(') + (fnc, args) = (func[:paren], func[paren + 1:-1]) + fncname = fnc + dct = fncdict.copy() + if 'name' in dct: + fncname = dct['name'] + del dct['name'] + self[fnc] = CfgFunction(name=fncname, arguments=args, **dct) + + def from_map(self, infuncs): + """Initialize the dictionary of functions by converting the input list + + :param infuncs: YAML list defining the functions + """ + for func in infuncs: + paren = func.find('(') + (fnc, args) = (func[:paren], func[paren + 1:-1]) + if fnc in self: + cfnc = self[fnc] + else: + self[fnc] = cfnc = CfgFunction(name=fnc, arguments=args) + for attr, val in list(infuncs[func].items()): + setattr(cfnc, attr, val) diff --git a/pyrseas/augment/schema.py b/pyrseas/augment/schema.py new file mode 100644 index 0000000..a095b64 --- /dev/null +++ b/pyrseas/augment/schema.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.schema + ~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, AugSchema and AugSchemaDict, derived from + DbAugment and DbAugmentDict, respectively. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.augment.table import AugTable + + +class AugSchema(DbAugment): + """A database schema definition, i.e., a named collection of tables, + views, triggers and other schema objects.""" + + keylist = ['name'] + + def apply(self, augdb): + """Augment objects in a schema. + + :param augdb: the augmenter dictionaries + """ + for tbl in self.tables: + self.tables[tbl].apply(augdb) + + def add_func(self, func): + """Add a function to the schema if not already present + + :param func: the possibly new function + """ + sch = self.current + if not hasattr(sch, 'functions'): + sch.functions = {} + if func.name not in sch.functions: + sch.functions.update({func.name: func}) + + +class AugSchemaDict(DbAugmentDict): + "The collection of schemas in a database" + + cls = AugSchema + + def from_map(self, augmap, augdb): + """Initialize the dictionary of schemas by converting the augmenter map + + :param augmap: the input YAML map defining the augmentations + :param augdb: collection of dictionaries defining the augmentations + + Starts the recursive analysis of the input map and + construction of the internal collection of dictionaries + describing the database objects. + """ + for key in augmap: + (objtype, spc, sch) = key.partition(' ') + if spc != ' ' or objtype != 'schema': + raise KeyError("Unrecognized object type: %s" % key) + schema = self[sch] = AugSchema(name=sch) + inschema = augmap[key] + augtables = {} + augfuncs = {} + for key in inschema: + if key.startswith('table '): + augtables.update({key: inschema[key]}) + elif key.startswith('function '): + augfuncs.update({key: inschema[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + augdb.tables.from_map(schema, augtables, augdb) + + def link_current(self, schemas): + """Connect schemas to be augmented to actual database schemas + + :param schemas: schemas in current database + """ + for sch in self: + if not sch in schemas: + raise KeyError("Schema %s not in current database" % sch) + if not hasattr(self[sch], 'current'): + self[sch].current = schemas[sch] + + def link_refs(self, dbtables): + """Connect tables and functions to their respective schemas + + :param dbtables: dictionary of tables + + Fills in the `tables` dictionary for each schema by + traversing the `dbtables` dictionary. + """ + for (sch, tbl) in dbtables: + table = dbtables[(sch, tbl)] + assert self[sch] + schema = self[sch] + if isinstance(table, AugTable): + if not hasattr(schema, 'tables'): + schema.tables = {} + schema.tables.update({tbl: table}) diff --git a/pyrseas/augment/table.py b/pyrseas/augment/table.py new file mode 100644 index 0000000..335034f --- /dev/null +++ b/pyrseas/augment/table.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.table + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines three classes: AugDbClass derived from + DbAugment, AugTable derived from AugDbClass, and AugClassDict + derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment + + +class AugDbClass(DbAugment): + """A table, sequence or view""" + + keylist = ['schema', 'name'] + + +class AugTable(AugDbClass): + """A database table definition""" + + def apply(self, augdb): + """Augment tables in a schema. + + :param augdb: the augmenter dictionaries + """ + currtbl = augdb.current.tables[self.current.key()] + if hasattr(self, 'audit_columns'): + if self.audit_columns not in augdb.auditcols: + raise KeyError("Specification %s not in current configuration" + % self.audit_columns) + augdb.auditcols[self.audit_columns].apply(currtbl, augdb) + + +class AugClassDict(DbAugmentDict): + "The collection of tables and similar objects in a database" + + cls = AugDbClass + + def from_map(self, schema, inobjs, augdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param augdb: collection of dictionaries defining the augmentations + """ + for k in inobjs: + (objtype, spc, key) = k.partition(' ') + if spc != ' ' or objtype not in ['table']: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'table': + self[(schema.name, key)] = table = AugTable( + schema=schema.name, name=key) + intable = inobjs[k] + if not intable: + raise ValueError("Table '%s' has no specification" % k) + for attr in intable: + if attr == 'audit_columns': + setattr(table, attr, intable[attr]) + else: + raise KeyError("Unrecognized attribute '%s' for %s" + % (attr, k)) + else: + raise KeyError("Unrecognized object type: %s" % k) + + def link_current(self, tables): + """Connect tables to be augmented to actual database tables + + :param tables: tables in current schema + """ + for (sch, tbl) in self: + if not (sch, tbl) in tables: + raise KeyError("Table %s.%s not in current database" % ( + sch, tbl)) + if not hasattr(self[(sch, tbl)], 'current'): + self[(sch, tbl)].current = tables[(sch, tbl)] diff --git a/pyrseas/augment/trigger.py b/pyrseas/augment/trigger.py new file mode 100644 index 0000000..7bbab9b --- /dev/null +++ b/pyrseas/augment/trigger.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augment.trigger + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: CfgTrigger derived from + DbAugment and CfgTriggerDict derived from DbAugmentDict. +""" +from pyrseas.augment import DbAugmentDict, DbAugment +from pyrseas.dbobject import split_schema_obj +from pyrseas.dbobject.trigger import Trigger + + +class CfgTrigger(DbAugment): + "A configuration trigger definition" + + keylist = ['name'] + + def apply(self, table): + """Create a trigger for the table passed in. + + :param table: table on which the trigger will be created + """ + newtrg = Trigger(self.name, table.schema, table.name, + getattr(self, 'description', None), + self.procedure, self.timing, self.level, self.events) + newtrg._iscfg = True + if newtrg.name.startswith('{{table_name}}'): + newtrg.name = newtrg.name.replace(newtrg.name[:14], table.name) + newtrg._table = table + if not hasattr(table, 'triggers'): + table.triggers = {} + if hasattr(newtrg, 'procedure'): + if newtrg.procedure.startswith('{{table_name}}'): + newtrg.procedure = newtrg.procedure.replace( + newtrg.procedure[:14], table.name) + (sch, fnc) = split_schema_obj(newtrg.procedure) + if sch != table.schema: + newtrg.procedure = "%s.%s" % (table.schema, fnc) + table.triggers.update({newtrg.name: newtrg}) + + +class CfgTriggerDict(DbAugmentDict): + "The collection of configuration triggers" + + cls = CfgTrigger + + def __init__(self, config): + """Initialize internal configuration triggers""" + for trg in config: + self[trg] = CfgTrigger(**config[trg]) + + def from_map(self, intrigs): + """Initialize the dictionary of triggers by converting the input dict + + :param intrigs: YAML dictionary defining the triggers + """ + for trg in intrigs: + if trg in self: + ctrg = self[trg] + else: + self[trg] = ctrg = CfgTrigger(name=trg) + for attr, val in list(intrigs[trg].items()): + setattr(ctrg, attr, val) diff --git a/pyrseas/augmentdb.py b/pyrseas/augmentdb.py new file mode 100644 index 0000000..30ad9d9 --- /dev/null +++ b/pyrseas/augmentdb.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.augmentdb + ~~~~~~~~~~~~~~~~~ + + An `AugmentDatabase` is initialized with a DbConnection object. + It consists of two "dictionary" container objects, each holding + various dictionary objects. The `db` Dicts object (inherited from + its parent class), defines the database schemas, including their + tables and other objects, by querying the system catalogs. The + `adb` AugDicts object defines the augmentation schemas and the + configuration objects based on the aug_map supplied to the `apply` + method. +""" +from pyrseas.database import Database +from pyrseas.dbobject.language import Language +from pyrseas.augment.schema import AugSchemaDict +from pyrseas.augment.table import AugClassDict +from pyrseas.augment.column import CfgColumnDict +from pyrseas.augment.function import CfgFunctionDict, CfgFunctionSourceDict +from pyrseas.augment.trigger import CfgTriggerDict +from pyrseas.augment.audit import CfgAuditColumnDict + + +def cfg_section(config, section): + "Return the configuration section if present, else an empty dict" + return config[section] if section in config else {} + + +class AugmentDatabase(Database): + """A database that is to be augmented""" + + class AugDicts(object): + """A holder for dictionaries (maps) describing augmentations""" + + def __init__(self, config): + """Initialize the various DbAugmentDict-derived dictionaries + + :param config: configuration dictionary + """ + self.schemas = AugSchemaDict() + self.tables = AugClassDict() + self.columns = CfgColumnDict(cfg_section(config, 'columns')) + self.funcsrcs = CfgFunctionSourceDict( + cfg_section(config, 'function_templates')) + self.functions = CfgFunctionDict(cfg_section(config, 'functions')) + self.triggers = CfgTriggerDict(cfg_section(config, 'triggers')) + self.auditcols = CfgAuditColumnDict( + cfg_section(config, 'audit_columns')) + + def _link_refs(self): + """Link related objects""" + self.schemas.link_refs(self.tables) + + def _link_current(self, db): + """Link augment objects to current catalog objects""" + self.current = db + self.schemas.link_current(db.schemas) + self.tables.link_current(db.tables) + + def add_func(self, schema, function): + """Add a function to a schema if not already present + + :param schema: schema name + :param function: the possibly new function + """ + if schema in self.schemas: + self.schemas[schema].add_func(function) + elif schema in self.current.schemas: + sch = self.current.schemas[schema] + if not hasattr(sch, 'functions'): + sch.functions = {} + if function.name not in sch.functions: + sch.functions.update({function.name: function}) + + def add_lang(self, lang): + """Add a language if not already present + + :param lang: the possibly new language + """ + if lang not in self.current.languages: + self.current.languages[lang] = Language(lang) + + def from_augmap(self, aug_map): + """Populate the augment objects from the input augment map + + :param aug_map: a YAML map defining the desired augmentations + + The `adb` holder is populated by various DbAugmentDict-derived + classes by traversing the YAML augmentation map. The objects + in the dictionary are then linked to related objects, e.g., + tables are linked to the schemas they belong. + """ + self.adb = self.AugDicts(cfg_section(self.config, 'augmenter')) + aug_schemas = {} + for key in aug_map: + if key == 'augmenter': + self._from_cfgmap(aug_map[key]) + elif key.startswith('schema '): + aug_schemas.update({key: aug_map[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + self.adb.schemas.from_map(aug_schemas, self.adb) + self.adb._link_refs() + self.adb._link_current(self.db) + + def _from_cfgmap(self, cfg_map): + """Populate configuration objects from the input configuration map + + :param cfg_map: a YAML map defining augmentation configuration + + The augmentations dictionary is populated by various + DbAugmentDict-derived classes by traversing the YAML + configuration map. + """ + for key in cfg_map: + if key == 'columns': + self.adb.columns.from_map(cfg_map[key]) + elif key in ['function_templates', 'function_segments']: + self.adb.funcsrcs.from_map(cfg_map[key]) + elif key == 'functions': + self.adb.functions.from_map(cfg_map[key]) + elif key == 'triggers': + self.adb.triggers.from_map(cfg_map[key]) + elif key == 'audit_columns': + self.adb.auditcols.from_map(cfg_map[key]) + else: + raise KeyError("Expected typed object, found '%s'" % key) + + def apply(self, aug_map): + """Apply augmentations to an existing database + + :param aug_map: a YAML map defining the desired augmentations + + Merges an existing database definition, as fetched from the + catalogs, with an input YAML defining augmentations on various + objects and an optional configuration map or the predefined + configuration. + """ + if not self.db: + self.from_catalog() + self.from_augmap(aug_map) + for sch in self.adb.schemas: + self.adb.schemas[sch].apply(self.adb) + return self.to_map() diff --git a/pyrseas/cmdargs.py b/pyrseas/cmdargs.py new file mode 100644 index 0000000..ef23796 --- /dev/null +++ b/pyrseas/cmdargs.py @@ -0,0 +1,110 @@ +# -*- coding: utf-8 -*- +"""Utility module for command line argument parsing""" + +import os +from argparse import ArgumentParser, FileType +import getpass + +import yaml + +from pyrseas.config import Config + +_cfg = None + +HELP_TEXT = { + 'host': "database server host or socket directory", + 'port': "database server port number", + 'username': "database user name" +} + + +def _help_dflt(arg, config): + kwdargs = {'help': HELP_TEXT[arg]} + if arg in config: + kwdargs['help'] += " (default %(default)s)" + kwdargs['default'] = config[arg] + return kwdargs + + +def _repo_path(cfg, key=None): + """Return path to root directory of repository or subdirectory + + :return: path + """ + repo = cfg['repository'] + if 'path' in repo: + path = repo['path'] + else: + path = os.getcwd() + subdir = '' if key is None else repo[key] + return os.path.normpath(os.path.join(path, subdir)) + + +def cmd_parser(description, version): + """Create command line argument parser with common PostgreSQL options + + :param description: text to display before the argument help + :param version: version of the caller + :return: the created parser + """ + global _cfg + + parent = ArgumentParser(add_help=False) + parent.add_argument('dbname', help='database name') + group = parent.add_argument_group('Connection options') + if _cfg is None: + _cfg = Config() + dbcfg = _cfg['database'] if 'database' in _cfg else {} + group.add_argument('-H', '--host', **_help_dflt('host', dbcfg)) + group.add_argument('-p', '--port', type=int, **_help_dflt('port', dbcfg)) + group.add_argument('-U', '--username', **_help_dflt('username', dbcfg)) + group.add_argument('-W', '--password', action="store_true", + help="force password prompt") + parent.add_argument('-c', '--config', type=FileType('r'), + help="configuration file path") + parent.add_argument('-r', '--repository', default=_repo_path(_cfg), + help="root of repository (default %(default)s)") + parent.add_argument('-o', '--output', type=FileType('w'), + help="output file name (default stdout)") + parser = ArgumentParser(parents=[parent], description=description) + parser.add_argument('--version', action='version', + version='%(prog)s ' + '%s' % version) + return parser + + +def parse_args(parser): + """Parse command line arguments and return configuration object + + :param parser: ArgumentParser created by cmd_parser + :return: a Configuration object + """ + arg_opts = parser.parse_args() + args = vars(arg_opts) + for key in ['database', 'files']: + if key not in _cfg: + _cfg[key] = {} + + def tfr(prim, key, val): + _cfg[prim][key] = val + del args[key] + + for key in ['dbname', 'host', 'port', 'username']: + tfr('database', key, args[key]) + tfr('database', 'password', + (getpass.getpass() if args['password'] else None)) + + for key in ['output', 'config']: + tfr('files', key, args[key]) + + if 'config' in _cfg['files'] and _cfg['files']['config']: + _cfg.merge(yaml.safe_load(_cfg['files']['config'])) + if 'repository' in args: + if args['repository'] != os.getcwd(): + _cfg['repository']['path'] = args['repository'] + del args['repository'] + + _cfg['files']['metadata_path'] = _repo_path(_cfg, 'metadata') + _cfg['files']['data_path'] = _repo_path(_cfg, 'data') + + _cfg['options'] = arg_opts + return _cfg diff --git a/pyrseas/config.py b/pyrseas/config.py new file mode 100644 index 0000000..93ab43a --- /dev/null +++ b/pyrseas/config.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +"""Utility module for configuration file parsing""" + +import os +import sys + +import yaml + + +CFG_FILE = os.environ.get("PYRSEAS_CONFIG_FILE", "config.yaml") + + +def _home_dir(): + if sys.platform == 'win32': + dir = os.getenv('APPDATA', '') + else: + dir = os.path.join(os.environ['HOME'], '.config') + return os.path.abspath(dir) + + +def _load_cfg(cfgdir): + cfgpath = '' + cfg = {} + if cfgdir is not None: + if os.path.isdir(cfgdir): + cfgpath = os.path.join(cfgdir, CFG_FILE) + elif os.path.isfile(cfgdir): + cfgpath = cfgdir + if os.path.exists(cfgpath): + with open(cfgpath) as f: + cfg = yaml.safe_load(f) + return cfg + + +class Config(dict): + "A configuration dictionary" + + def __init__(self, sys_only=False): + self.update(_load_cfg( + os.environ.get("PYRSEAS_SYS_CONFIG", os.path.abspath(os.path.join( + os.path.dirname(__file__)))))) + if sys_only: + return + self.merge(_load_cfg(os.environ.get("PYRSEAS_USER_CONFIG", + os.path.join(_home_dir(), 'pyrseas')))) + if 'repository' in self and 'path' in self['repository']: + cfgpath = self['repository']['path'] + else: + cfgpath = os.getcwd() + self.merge(_load_cfg(cfgpath)) + + def merge(self, cfg): + """Merge extra configuration + + :param cfg: extra configuration (dict) + """ + for key, val in list(cfg.items()): + if key in self: + self[key].update(val) + else: + self[key] = val diff --git a/pyrseas/config.yaml b/pyrseas/config.yaml new file mode 100644 index 0000000..018af66 --- /dev/null +++ b/pyrseas/config.yaml @@ -0,0 +1,160 @@ +augmenter: + audit_columns: + created_date_only: + columns: + - created_date + default: + columns: + - modified_by_user + - modified_timestamp + triggers: + - audit_default + modified_only: + columns: + - modified_timestamp + triggers: + - audit_modified_only + full: + columns: + - created_by_user + - created_by_ip_address + - created_timestamp + - modified_by_user + - modified_by_ip_address + - modified_timestamp + triggers: + - audit_full + columns: + created_by_ip_address: + not_null: true + type: inet + created_by_user: + not_null: true + type: character varying(63) + created_date: + default: ('now'::text)::date + not_null: true + type: date + created_timestamp: + not_null: true + type: timestamp with time zone + modified_by_ip_address: + not_null: true + type: inet + modified_by_user: + not_null: true + type: character varying(63) + modified_timestamp: + not_null: true + type: timestamp with time zone + function_templates: + functempl_audit_default: |2- + + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + RETURN NEW; + END + functempl_audit_full: |2- + + DECLARE + user_name name := pyrseas.get_session_variable('audit_user', + SESSION_USER::varchar); + ip_address inet := pyrseas.get_session_variable('audit_ip_address', + COALESCE(pg_catalog.inet_client_addr()::varchar, '0.0.0.0')); + BEGIN + IF TG_OP = 'INSERT' THEN + NEW.{{created_timestamp}} = CURRENT_TIMESTAMP; + NEW.{{created_by_user}} = user_name; + NEW.{{created_by_ip_address}} = ip_address; + ELSIF TG_OP = 'UPDATE' THEN + NEW.{{created_timestamp}} = OLD.created_timestamp; + NEW.{{created_by_user}} = OLD.created_by_user; + NEW.{{created_by_ip_address}} = OLD.created_by_ip_address; + END IF; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + NEW.{{modified_by_user}} = user_name; + NEW.{{modified_by_ip_address}} = ip_address; + RETURN NEW; + END + functempl_audit_modified: |2- + + BEGIN + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP; + RETURN NEW; + END + functions: + audit_default(): + description: |- + Provides modified_by_user and modified_timestamp values for audit + columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_default}}' + audit_full(): + description: |- + Provides created/modified values for audit columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_full}}' + audit_modified(): + description: |- + Provides modified_timestamp values for audit columns. + language: plpgsql + returns: trigger + security_definer: true + source: '{{functempl_audit_modified}}' + schema pyrseas: + function get_session_variable(var_name character varying): + language: plpgsql + returns: character varying + source: |2 + BEGIN + RETURN pg_catalog.current_setting('pyrseas.' || var_name); + END; + function get_session_variable(var_name character varying, default_value character varying): + language: plpgsql + returns: character varying + source: |2 + BEGIN + RETURN pg_catalog.current_setting('pyrseas.' || var_name); + EXCEPTION + WHEN undefined_object THEN + RETURN default_value; + END; + function set_session_variable(var_name character varying, var_value character varying): + language: plpgsql + returns: void + source: |2 + BEGIN + PERFORM pg_catalog.set_config('pyrseas.' || var_name, var_value, false); + END; + triggers: + audit_default: + events: + - update + level: row + name: '{{table_name}}_20_audit_default' + procedure: audit_default() + timing: before + audit_full: + events: + - insert + - update + level: row + name: '{{table_name}}_20_audit_full' + procedure: audit_full() + timing: before + audit_modified_only: + events: + - insert + - update + level: row + name: '{{table_name}}_20_audit_modified_only' + procedure: audit_modified() + timing: before +repository: + metadata: metadata + data: metadata diff --git a/pyrseas/database.py b/pyrseas/database.py new file mode 100644 index 0000000..104335f --- /dev/null +++ b/pyrseas/database.py @@ -0,0 +1,650 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.database + ~~~~~~~~~~~~~~~~ + + A `Database` is initialized with a DbConnection object. It + consists of one or two `Dicts` objects, each holding various + dictionary objects. The `db` Dicts object defines the database + schemas, including their tables and other objects, by querying the + system catalogs. The `ndb` Dicts object defines the schemas based + on the `input_map` supplied to the `from_map` method. +""" +import os +import sys +from operator import itemgetter +from collections import defaultdict, deque +import yaml + +from pyrseas.lib.dbconn import DbConnection + +from pyrseas.yamlutil import yamldump +from pyrseas.dbobject import fetch_reserved_words, DbObjectDict, DbSchemaObject +from pyrseas.dbobject.language import LanguageDict +from pyrseas.dbobject.cast import CastDict +from pyrseas.dbobject.schema import SchemaDict +from pyrseas.dbobject.dbtype import TypeDict +from pyrseas.dbobject.table import ClassDict +from pyrseas.dbobject.column import ColumnDict +from pyrseas.dbobject.constraint import ConstraintDict +from pyrseas.dbobject.index import IndexDict +from pyrseas.dbobject.function import ProcDict +from pyrseas.dbobject.operator import OperatorDict +from pyrseas.dbobject.operclass import OperatorClassDict +from pyrseas.dbobject.operfamily import OperatorFamilyDict +from pyrseas.dbobject.rule import RuleDict +from pyrseas.dbobject.trigger import TriggerDict +from pyrseas.dbobject.conversion import ConversionDict +from pyrseas.dbobject.textsearch import TSConfigurationDict, TSDictionaryDict +from pyrseas.dbobject.textsearch import TSParserDict, TSTemplateDict +from pyrseas.dbobject.foreign import ForeignDataWrapperDict +from pyrseas.dbobject.foreign import ForeignServerDict, UserMappingDict +from pyrseas.dbobject.foreign import ForeignTableDict +from pyrseas.dbobject.extension import ExtensionDict +from pyrseas.dbobject.collation import CollationDict +from pyrseas.dbobject.eventtrig import EventTriggerDict + + +def flatten(lst): + "Flatten a list possibly containing lists to a single list" + for elem in lst: + if isinstance(elem, list) and not isinstance(elem, str): + for subelem in flatten(elem): + yield subelem + else: + yield elem + + +class CatDbConnection(DbConnection): + """A database connection, specialized for querying catalogs""" + + def connect(self): + """Connect to the database""" + super(CatDbConnection, self).connect() + schs = self.fetchall("SELECT current_schemas(false)") + addschs = [sch for sch in schs[0]["current_schemas"] if sch != "public"] + srch_path = "pg_catalog" + if addschs: + srch_path += ", " + ", ".join(addschs) + self.execute("set search_path to %s" % srch_path) + self.commit() + self._version = self.conn.info.server_version + + @property + def version(self): + "The server's version number" + if self.conn is None: + self.connect() + return self._version + + +class Database(object): + """A database definition, from its catalogs and/or a YAML spec.""" + + class Dicts(object): + """A holder for dictionaries (maps) describing a database""" + + def __init__(self, dbconn=None, single_db=False): + """Initialize the various DbObjectDict-derived dictionaries + + :param dbconn: a DbConnection object + """ + self.schemas = SchemaDict(dbconn) + self.extensions = ExtensionDict(dbconn) + self.languages = LanguageDict(dbconn) + self.casts = CastDict(dbconn) + self.types = TypeDict(dbconn) + self.tables = ClassDict(dbconn) + self.columns = ColumnDict(dbconn) + self.constraints = ConstraintDict(dbconn) + self.indexes = IndexDict(dbconn) + self.functions = ProcDict(dbconn) + self.operators = OperatorDict(dbconn) + self.operclasses = OperatorClassDict(dbconn) + self.operfams = OperatorFamilyDict(dbconn) + self.rules = RuleDict(dbconn) + self.triggers = TriggerDict(dbconn) + self.conversions = ConversionDict(dbconn) + self.tstempls = TSTemplateDict(dbconn) + self.tsdicts = TSDictionaryDict(dbconn) + self.tsparsers = TSParserDict(dbconn) + self.tsconfigs = TSConfigurationDict(dbconn) + self.fdwrappers = ForeignDataWrapperDict(dbconn) + self.servers = ForeignServerDict(dbconn) + self.usermaps = UserMappingDict(dbconn) + self.ftables = ForeignTableDict(dbconn) + self.collations = CollationDict(dbconn) + self.eventtrigs = EventTriggerDict(dbconn) + + # Populate a map from system catalog to the respective dict + self._catalog_map = {} + for _, d in self.all_dicts(single_db): + if d.cls.catalog is not None: + self._catalog_map[d.cls.catalog] = d + + # Map from objects extkey to their (dict name, key) + self._extkey_map = {} + + def _get_by_extkey(self, extkey): + """Return any database item from its extkey + + Note: probably doesn't work for all the objects, e.g. constraints + may clash because two in different tables have different extkeys. + However this shouldn't matter as such objects are generated as part + of the containing one and they should be returned by the + `get_implied_deps()` implementation of specific classes (which + would look for the object in by key in the right dict instead, + (e.g. check `Domain.get_implied_deps()` implementation. + + """ + try: + return self._extkey_map[extkey] + except KeyError: + # TODO: Likely it's the first time we call this function so + # let's warm up the cache. But we should really define the life + # cycle of this object as trying and catching KeyError on it is + # *very* expensive! + for _, d in self.all_dicts(): + for obj in list(d.values()): + self._extkey_map[obj.extern_key()] = obj + + return self._extkey_map[extkey] + + def all_dicts(self, non_empty=False): + """Iterate over the DbObjectDict-derived dictionaries returning + an ordered list of tuples (dict name, DbObjectDict object). + + :param non_empty: do not include empty dicts + + :return: list of tuples + """ + rv = [] + for attr in self.__dict__: + d = getattr(self, attr) + if non_empty and len(d) == 0: + continue + if isinstance(d, DbObjectDict): + # skip ColumnDict as not needed for dependency tracking + # and internally has lists, not objects + if not isinstance(d, ColumnDict): + rv.append((attr, d)) + + # first return the dicts for non-schema objects, then the + # others, each group sorted alphabetically. + rv.sort(key=lambda pair: (issubclass(pair[1].cls, DbSchemaObject), + pair[1].cls.__name__)) + + return rv + + def dbobjdict_from_catalog(self, catalog): + """Given a catalog name, return corresponding DbObjectDict + + :param catalog: full name of a pg_ catalog + :return: DbObjectDict object + """ + return self._catalog_map.get(catalog) + + def find_type(self, name): + """Return a db type given a qualname + + Note that tables and views are types too. + """ + rv = self.types.find(name) + if rv is not None: + return rv + + rv = self.tables.find(name) + return rv + + def __init__(self, config): + """Initialize the database + + :param config: configuration dictionary + """ + db = config['database'] + self.dbconn = CatDbConnection(db['dbname'], db['username'], + db['password'], db['host'], db['port']) + self.db = None + self.config = config + + def _link_refs(self, db): + """Link related objects""" + langs = [] + if self.dbconn.version >= 90100: + langs = [lang["lanname"] for lang in self.dbconn.fetchall( + """SELECT lanname FROM pg_language l + JOIN pg_depend p ON (l.oid = p.objid) + WHERE deptype = 'e' """)] + db.languages.link_refs(db.functions, langs) + copycfg = {} + if 'datacopy' in self.config: + copycfg = self.config['datacopy'] + db.schemas.link_refs(db, copycfg) + db.tables.link_refs(db.columns, db.constraints, db.indexes, db.rules, + db.triggers) + db.functions.link_refs(db.types) + db.fdwrappers.link_refs(db.servers) + db.servers.link_refs(db.usermaps) + db.ftables.link_refs(db.columns) + db.types.link_refs(db.columns, db.constraints, db.functions) + db.constraints.link_refs(db) + + def _build_dependency_graph(self, db, dbconn): + """Build the dependency graph of the database objects + + :param db: dictionary of dictionary of all objects + :param dbconn: a DbConnection object + """ + alldeps = defaultdict(list) + + # This query wanted to be simple. it got complicated because + # we don't handle indexes together with the other pg_class + # but in their own pg_index place (so fetch i1, i2) + # "Normal" dependencies, but excluding system objects + # (objid < 16384 and refobjid < 16384) + query = """SELECT DISTINCT + CASE WHEN i1.indexrelid IS NOT NULL + THEN 'pg_index'::regclass + ELSE classid::regclass END AS class_name, objid, + CASE WHEN i2.indexrelid IS NOT NULL + THEN 'pg_index'::regclass + ELSE refclassid::regclass END AS refclass, refobjid + FROM pg_depend + LEFT JOIN pg_index i1 ON classid = 'pg_class'::regclass + AND objid = i1.indexrelid + LEFT JOIN pg_index i2 + ON refclassid = 'pg_class'::regclass + AND refobjid = i2.indexrelid + WHERE deptype = 'n' + AND NOT (objid < 16384 AND refobjid < 16384)""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['objid']].append( + (r['refclass'], r['refobjid'])) + + # The dependencies across views is not in pg_depend. We have to + # parse the rewrite rule. "ev_class >= 16384" is to exclude + # system views. + query = r"""SELECT DISTINCT 'pg_class' AS class_name, ev_class, + CASE WHEN depid[1] = 'relid' THEN 'pg_class' + WHEN depid[1] = 'funcid' THEN 'pg_proc' + END AS refclass, depid[2]::oid AS refobjid + FROM (SELECT ev_class, regexp_matches(ev_action, + ':(relid|funcid)\s+(\d+)', 'g') AS depid + FROM pg_rewrite + WHERE rulename = '_RETURN' + AND ev_class >= 16384) x + LEFT JOIN pg_class c + ON (depid[1], depid[2]::oid) = ('relid', c.oid) + LEFT JOIN pg_namespace cs ON cs.oid = relnamespace + LEFT JOIN pg_proc p + ON (depid[1], depid[2]::oid) = ('funcid', p.oid) + LEFT JOIN pg_namespace ps ON ps.oid = pronamespace + WHERE ev_class <> depid[2]::oid + AND coalesce(cs.nspname, ps.nspname) + NOT IN ('information_schema', 'pg_catalog')""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['ev_class']].append( + (r['refclass'], r['refobjid'])) + + # Add the dependencies between a table and other objects through the + # columns defaults + query = """SELECT 'pg_class' AS class_name, adrelid, + d.refclassid::regclass, d.refobjid + FROM pg_attrdef ad JOIN pg_depend d + ON classid = 'pg_attrdef'::regclass AND objid = ad.oid + AND deptype = 'n'""" + for r in dbconn.fetchall(query): + alldeps[r['class_name'], r['adrelid']].append( + (r['refclassid'], r['refobjid'])) + + for (stbl, soid), deps in list(alldeps.items()): + sdict = db.dbobjdict_from_catalog(stbl) + if sdict is None or len(sdict) == 0: + continue + src = sdict.by_oid.get(soid) + if src is None: + continue + for ttbl, toid in deps: + tdict = db.dbobjdict_from_catalog(ttbl) + if tdict is None or len(tdict) == 0: + continue + tgt = tdict.by_oid.get(toid) + if tgt is None: + continue + src.depends_on.append(tgt) + + def _trim_objects(self, schemas): + """Remove unwanted schema objects + + :param schemas: list of schemas to keep + """ + for objtype in ['types', 'tables', 'constraints', 'indexes', + 'functions', 'operators', 'operclasses', 'operfams', + 'rules', 'triggers', 'conversions', 'tstempls', + 'tsdicts', 'tsparsers', 'tsconfigs', 'extensions', + 'collations', 'eventtrigs']: + objdict = getattr(self.db, objtype) + for obj in list(objdict.keys()): + # obj[0] is the schema name in all these dicts + if obj[0] not in schemas: + del objdict[obj] + for sch in list(self.db.schemas.keys()): + if sch not in schemas: + del self.db.schemas[sch] + # exclude database-wide objects + self.db.languages = LanguageDict() + self.db.casts = CastDict() + + def from_catalog(self, single_db=False): + """Populate the database objects by querying the catalogs + + :param single_db: populating only this database? + + The `db` holder is populated by various DbObjectDict-derived + classes by querying the catalogs. A dependency graph is + constructed by querying the pg_depend catalog. The objects in + the dictionary are then linked to related objects, e.g., + columns are linked to the tables they belong. + """ + self.db = self.Dicts(self.dbconn, single_db) + self._build_dependency_graph(self.db, self.dbconn) + if self.dbconn.conn: + self.dbconn.conn.close() + self._link_refs(self.db) + + def from_map(self, input_map, langs=None): + """Populate the new database objects from the input map + + :param input_map: a YAML map defining the new database + :param langs: list of language templates + + The `ndb` holder is populated by various DbObjectDict-derived + classes by traversing the YAML input map. The objects in the + dictionary are then linked to related objects, e.g., columns + are linked to the tables they belong. + """ + self.ndb = self.Dicts() + input_schemas = {} + input_extens = {} + input_langs = {} + input_casts = {} + input_fdws = {} + input_ums = {} + input_evttrigs = {} + for key in input_map: + if key.startswith('schema '): + input_schemas.update({key: input_map[key]}) + elif key.startswith('extension '): + input_extens.update({key: input_map[key]}) + elif key.startswith('language '): + input_langs.update({key: input_map[key]}) + elif key.startswith('cast '): + input_casts.update({key: input_map[key]}) + elif key.startswith('foreign data wrapper '): + input_fdws.update({key: input_map[key]}) + elif key.startswith('user mapping for '): + input_ums.update({key: input_map[key]}) + elif key.startswith('event trigger '): + input_evttrigs.update({key: input_map[key]}) + else: + raise KeyError("Expected typed object, found '%s'" % key) + self.ndb.extensions.from_map(input_extens, self.ndb) + self.ndb.languages.from_map(input_langs) + self.ndb.schemas.from_map(input_schemas, self.ndb) + self.ndb.casts.from_map(input_casts, self.ndb) + self.ndb.fdwrappers.from_map(input_fdws, self.ndb) + self.ndb.eventtrigs.from_map(input_evttrigs, self.ndb) + self._link_refs(self.ndb) + + def map_from_dir(self): + """Read the database maps starting from the metadata directory + + :return: dictionary + """ + metadata_dir = self.config['files']['metadata_path'] + if not os.path.isdir(metadata_dir): + sys.exit("Metadata directory '%s' doesn't exist" % metadata_dir) + + def load(subdir, obj): + with open(os.path.join(subdir, obj), 'r') as f: + objmap = yaml.safe_load(f) + return objmap if isinstance(objmap, dict) else {} + + inmap = {} + for entry in os.listdir(metadata_dir): + if entry.endswith('.yaml'): + if entry.startswith('database.'): + continue + if not entry.startswith('schema.'): + inmap.update(load(metadata_dir, entry)) + else: + # skip over unknown files/dirs + if not entry.startswith('schema.'): + continue + # read schema.xxx.yaml first + schmap = load(metadata_dir, entry + '.yaml') + assert(len(schmap) == 1) + key = list(schmap.keys())[0] + inmap.update({key: {}}) + subdir = os.path.join(metadata_dir, entry) + if os.path.isdir(subdir): + for schobj in os.listdir(subdir): + schmap[key].update(load(subdir, schobj)) + inmap.update(schmap) + + return inmap + + def to_map(self, quote_reserved=True): + """Convert the db maps to a single hierarchy suitable for YAML + + :param quote_reserved: fetch reserved words + :return: a YAML-suitable dictionary (without any Python objects) + """ + if not self.db: + self.from_catalog(True) + + opts = self.config['options'] + + def mkdir_parents(dir): + head, tail = os.path.split(dir) + if head and not os.path.isdir(head): + mkdir_parents(head) + if tail: + os.mkdir(dir) + + if opts.multiple_files: + opts.metadata_dir = self.config['files']['metadata_path'] + if not os.path.exists(opts.metadata_dir): + mkdir_parents(opts.metadata_dir) + dbfilepath = os.path.join(opts.metadata_dir, 'database.%s.yaml' % + self.dbconn.dbname) + if os.path.exists(dbfilepath): + with open(dbfilepath, 'r') as f: + objmap = yaml.safe_load(f) + for obj, val in list(objmap.items()): + if isinstance(val, dict): + dirpath = '' + for schobj, fpath in list(val.items()): + filepath = os.path.join(opts.metadata_dir, fpath) + if os.path.exists(filepath): + os.remove(filepath) + if schobj == 'schema': + (dirpath, ext) = os.path.splitext(filepath) + if os.path.exists(dirpath): + os.rmdir(dirpath) + else: + filepath = os.path.join(opts.metadata_dir, val) + if (os.path.exists(filepath)): + os.remove(filepath) + if quote_reserved: + fetch_reserved_words(self.dbconn) + + dbmap = self.db.extensions.to_map(self.db, opts) + dbmap.update(self.db.languages.to_map(self.db, opts)) + dbmap.update(self.db.casts.to_map(self.db, opts)) + dbmap.update(self.db.fdwrappers.to_map(self.db, opts)) + dbmap.update(self.db.eventtrigs.to_map(self.db, opts)) + if 'datacopy' in self.config: + opts.data_dir = self.config['files']['data_path'] + if not os.path.exists(opts.data_dir): + mkdir_parents(opts.data_dir) + dbmap.update(self.db.schemas.to_map(self.db, opts)) + + if opts.multiple_files: + with open(dbfilepath, 'w') as f: + f.write(yamldump(dbmap)) + + return dbmap + + def diff_map(self, input_map, quote_reserved=True): + """Generate SQL to transform an existing database + + :param input_map: a YAML map defining the new database + :param quote_reserved: fetch reserved words + :return: list of SQL statements + + Compares the existing database definition, as fetched from the + catalogs, to the input YAML map and generates SQL statements + to transform the database into the one represented by the + input. + """ + from .dbobject.table import Table + + if not self.db: + self.from_catalog() + opts = self.config['options'] + if opts.schemas: + schlist = ['schema ' + sch for sch in opts.schemas] + for sch in list(input_map.keys()): + if sch not in schlist and sch.startswith('schema '): + del input_map[sch] + self._trim_objects(opts.schemas) + + # quote_reserved is only set to False by most tests + if quote_reserved: + fetch_reserved_words(self.dbconn) + + self.from_map(input_map) + if opts.revert: + (self.db, self.ndb) = (self.ndb, self.db) + del self.ndb.schemas['pg_catalog'] + self.db.languages.dbconn = self.dbconn + + # First sort the objects in the new db in dependency order + new_objs = [] + for _, d in self.ndb.all_dicts(): + pairs = list(d.items()) + pairs.sort() + new_objs.extend(list(map(itemgetter(1), pairs))) + + new_objs = self.dep_sorted(new_objs, self.ndb) + + # Then generate the sql for all the objects, walking in dependency + # order over all the db objects + + stmts = [] + for new in new_objs: + d = self.db.dbobjdict_from_catalog(new.catalog) + old = d.get(new.key()) + if old is not None: + stmts.append(old.alter(new)) + else: + stmts.append(new.create_sql(self.dbconn.version)) + + # Check if the object just created was renamed, in which case + # don't try to delete the original one + if getattr(new, 'oldname', None): + try: + origname, new.name = new.name, new.oldname + oldkey = new.key() + finally: + new.name = origname + # Intentionally raising KeyError as tested e.g. in + # test_bad_rename_view -- ok Joe? + old = d[oldkey] + old._nodrop = True + + # Order the old database objects in reverse dependency order + old_objs = [] + for _, d in self.db.all_dicts(): + pairs = list(d.items()) + pairs.sort + old_objs.extend(list(map(itemgetter(1), pairs))) + old_objs = self.dep_sorted(old_objs, self.db) + old_objs.reverse() + + # Drop the objects that don't appear in the new db + for old in old_objs: + d = self.ndb.dbobjdict_from_catalog(old.catalog) + if isinstance(old, Table): + new = d.get(old.key()) + if new is not None: + stmts.extend(old.alter_drop_columns(new)) + if not getattr(old, '_nodrop', False) and old.key() not in d: + stmts.extend(old.drop()) + + if 'datacopy' in self.config: + opts.data_dir = self.config['files']['data_path'] + stmts.append(self.ndb.schemas.data_import(opts)) + + stmts = [s for s in flatten(stmts)] + funcs = False + for s in stmts: + if "LANGUAGE sql" in s and ( + s.startswith("CREATE FUNCTION ") or + s.startswith("CREATE OR REPLACE FUNCTION ")): + funcs = True + break + if funcs: + stmts.insert(0, "SET check_function_bodies = false") + + return stmts + + def dep_sorted(self, objs, db): + """Sort `objs` in order of dependency. + + The function implements the classic Kahn 62 algorighm, see + . + """ + # List of objects to return + L = [] + + # Collect the graph edges. + # Note that our "dependencies" are sort of backwards compared to the + # terms used in the algorithm (an edge in the algo would be from the + # schema to the table, we have the table depending on the schema) + ein = defaultdict(set) + eout = defaultdict(deque) + for obj in objs: + for dep in obj.get_deps(db): + eout[dep].append(obj) + ein[obj].add(dep) + + # The objects with no dependency to start with + S = deque() + for obj in objs: + if obj not in ein: + S.append(obj) + + while S: + # Objects with no dependencies can be emitted + obj = S.popleft() + L.append(obj) + + # Delete the edges and check if depending objects have no + # dependency now + while eout[obj]: + ch = eout[obj].popleft() + ein[ch].remove(obj) + if not ein[ch]: + del ein[ch] + S.append(ch) + + del eout[obj] # remove the empty set + + assert bool(ein) == bool(eout) + if not ein: + return L + else: + # is it possible? How do we deal with that? + raise Exception("the objects dependencies graph has loops") diff --git a/pyrseas/dbaugment.py b/pyrseas/dbaugment.py new file mode 100644 index 0000000..e655448 --- /dev/null +++ b/pyrseas/dbaugment.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""dbaugment - Augment a PostgreSQL database""" + +from __future__ import print_function +import sys +from argparse import FileType + +import yaml + +from pyrseas import __version__ +from pyrseas.yamlutil import yamldump +from pyrseas.augmentdb import AugmentDatabase +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(): + """Augment database specifications""" + parser = cmd_parser("Generate a modified schema for a PostgreSQL " + "database, in YAML format, augmented with specified " + "attributes and procedures", __version__) + # TODO: processing of multiple files, owner and privileges + parser.add_argument('-m', '--multiple-files', action='store_true', + help='multiple files (metadata directory)') + parser.add_argument('-O', '--no-owner', action='store_true', + help='exclude object ownership information') + parser.add_argument('-x', '--no-privileges', action='store_true', + dest='no_privs', + help='exclude privilege (GRANT/REVOKE) information') + parser.add_argument('spec', nargs='?', type=FileType('r'), + default=sys.stdin, help='YAML augmenter specification') + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + augdb = AugmentDatabase(cfg) + augmap = yaml.safe_load(options.spec) + try: + outmap = augdb.apply(augmap) + except BaseException as exc: + if type(exc) != KeyError: + raise + sys.exit("ERROR: %s" % str(exc)) + print(yamldump(outmap), file=output or sys.stdout) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/dbobject/__init__.py b/pyrseas/dbobject/__init__.py new file mode 100644 index 0000000..4abd59f --- /dev/null +++ b/pyrseas/dbobject/__init__.py @@ -0,0 +1,680 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject + ~~~~~~~~~~~~~~~~ + + This defines two low level classes and an intermediate class. + Most Pyrseas classes are derived from either DbObject or + DbObjectDict. +""" +import os +import re +import string +from functools import wraps + +from pyrseas.yamlutil import yamldump +from .privileges import privileges_to_map, add_grant, diff_privs +from .privileges import privileges_from_map + + +VALID_FIRST_CHARS = string.ascii_lowercase + '_' +VALID_CHARS = string.ascii_lowercase + string.digits + '_$' +RESERVED_WORDS = [] +NON_FILENAME_CHARS = re.compile(r'\W', re.U) +MAX_PG_IDENT_LEN = 63 +MAX_IDENT_LEN = int(os.environ.get("PYRSEAS_MAX_IDENT_LEN", 32)) + + +def fetch_reserved_words(db): + """Fetch PostgreSQL reserved words + + :param db: DbConnection object + """ + global RESERVED_WORDS + + if len(RESERVED_WORDS) == 0: + RESERVED_WORDS = [word["word"] for word in + db.fetchall("""SELECT word FROM pg_get_keywords() + WHERE catcode != 'U'""")] + + +def quote_id(name): + """Quotes an identifier if necessary. + + :param name: string to be quoted + + :return: possibly quoted string + """ + regular_id = True + if not name[0] in VALID_FIRST_CHARS or name in RESERVED_WORDS: + regular_id = False + else: + for ltr in name[1:]: + if ltr not in VALID_CHARS: + regular_id = False + break + + return regular_id and name or '"%s"' % name + + +def split_schema_obj(obj, sch=None): + """Return a (schema, object) tuple given a possibly schema-qualified name + + :param obj: object name or schema.object + :param sch: schema name (defaults to 'pg_catalog') + :return: tuple + """ + def undelim(ident): + if ident[0] == '"' and ident[-1] == '"': + ident = ident[1:-1] + return ident + + qualsch = sch + if sch is None: + qualsch = 'pg_catalog' + if obj[0] == '"' and obj[-1] == '"': + if '"."' in obj: + (qualsch, obj) = obj.split('"."') + qualsch = qualsch[1:] + obj = obj[:-1] + else: + obj = obj[1:-1] + else: + # TODO: properly handle functions + if '.' in obj and '(' not in obj: + (qualsch, obj) = obj.split('.') + if sch != qualsch: + sch = qualsch + return (undelim(sch), undelim(obj)) + + +def split_func_args(obj): + """Split function name and argument from a signature, e.g. fun(int, text) + + :param obj: The string to parse + :return: 2-item tuple (name, args), args is a list of strings. + + TODO: make it safer against pathologic input (names containing' '( and ',') + """ + tokens = obj.split('(') + if len(tokens) != 2 or not tokens[1].endswith(')'): + raise ValueError("not a valid function signature: '%s'" % obj) + name = tokens[0] + args = [arg.strip() for arg in tokens[1][:-1].split(',')] + return name, args + + +def commentable(func): + """Decorator to add comments to various objects""" + @wraps(func) + def add_comment(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if obj.description is not None: + stmts.append(obj.comment()) + return stmts + return add_comment + + +def grantable(func): + """Decorator to add GRANT to various objects""" + @wraps(func) + def grant(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if hasattr(obj, 'privileges'): + for priv in obj.privileges: + stmts.append(add_grant(obj, priv)) + return stmts + return grant + + +def ownable(func): + """Decorator to add ALTER OWNER to various objects""" + @wraps(func) + def add_alter(obj, *args, **kwargs): + stmts = func(obj, *args, **kwargs) + if hasattr(obj, 'owner'): + stmts.append(obj.alter_owner()) + return stmts + return add_alter + + +class DbObject(object): + "A single object in a database catalog, e.g., a schema, a table, a column" + + keylist = ['name'] + """List of attributes that uniquely identify the object in the catalogs + + See description of :meth:`key` for further details. + """ + + @property + def objtype(self): + """Type of object as an uppercase string, for SQL syntax generation + + This is used in most CREATE, ALTER and DROP statements. It is + also used by :meth:`extern_key` in lowercase form. + """ + if self._objtype is None: + self._objtype = self.__class__.__name__.upper() + return self._objtype + + catalog = None + """The name of the system catalog where these objects live + """ + + allprivs = '' + + def __init__(self, name, description=None, **attrs): + """Initialize the catalog object from a dictionary of attributes + + :param name: name of object + :param description: comment text describing object + :param attrs: the dictionary of attributes + + Non-key attributes without a value are discarded. Values that + are multi-line strings are treated specially so that YAML will + output them in block style. + """ + self.name = name + self.description = description + self.depends_on = [] + self.owner = None + self.privileges = [] + self._objtype = None + + def _init_own_privs(self, owner=None, privileges=[]): + """Initialize owner and privileges attributes + + :param owner: name of user that owns the object +- :param privileges: privileges on object + + The vast majority of Postgres database objects have owner and + privileges attributes. Hence all base DbObject instances have + those attributes. This method allows separate initialization. + """ + self.owner = owner + if isinstance(privileges, str): + privileges = privileges.split(',') + self.privileges = privileges or [] + + def __repr__(self): + return "<%s at 0x%x>" % (self.extern_key(), id(self)) + + # hash and eq allow to use the objects as dict keys + def __hash__(self): + return hash((self.__class__, self.key())) + + def __eq__(self, other): + if self.__class__ is other.__class__: + return self.key() == other.key() + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + @staticmethod + def query(dbversion=None): + """The SQL SELECT query to fetch object instances from the catalogs + + :param dbversion: Postgres version identifier + + This is used by the method :meth:`fetch`. The `dbversion` + parameter is used in descendant classes to customize the + queries according to the target Postgres version. + """ + return "" + + def extern_key(self): + """Return the key to be used in external maps for this object + + :return: string + + This is used for the first two levels of external maps. The + first level is the one that includes schemas, as well as + extensions, languages, casts and foreign data wrappers. The + second level includes all schema-owned objects, i.e., tables, + functions, operators, etc. All subsequent levels, e.g., + primary keys, indexes, etc., currently use the object name as + the external identifier, appearing in the map after an object + grouping header, such as ``primary_key``. + + The common format for an external key is `object-type + non-schema-qualified-name`, where `object-type` is the + lowercase version of :attr:`objtype`, e.g., ``table + tablename``. Some object types require more, e.g., functions + need the signature, so they override this implementation. + + """ + return '%s %s' % (self.objtype.lower(), self.name) + + def extern_filename(self, ext='yaml', truncate=False): + """Return a filename to be used to output external files + + :param ext: file extension + :param truncate: truncate filename to MAX_IDENT_LEN + :return: filename string + + This is used for the first two levels of external (metadata) + files. The first level is the one that includes schemas, as + well as extensions, languages, casts and FDWs. The second + level includes all schema-owned objects, i.e., tables, + functions, operators, etc. + + The common format for the filename is `objtype.objname.yaml`, + e.g., for a table `t1` the filename is "table.t1.yaml". For + an object name that has characters not allowed in filesystems, + the characters are replaced by underscores. + + """ + max_len = MAX_IDENT_LEN if truncate else MAX_PG_IDENT_LEN + + def xfrm_filename(objtype, objid=None): + """Generic transformation of object identifier to a filename + + :param objtype: object type + :param objid: object identifier, usually the 'name' attribute + :return: filename string + """ + if objid: + filename = '%s.%.*s.%s' % ( + objtype, max_len, re.sub(NON_FILENAME_CHARS, '_', objid), + ext) + else: + filename = '%s.%s' % (objtype.replace(' ', '_'), ext) + return filename.lower() + + if hasattr(self, 'single_extern_file') and self.single_extern_file: + return xfrm_filename(self.objtype) + + return xfrm_filename(self.objtype, self.name) + + def key(self): + """Return a tuple that identifies the database object + + :return: a single string or a tuple of strings + + This is used as key for all internal maps. The first-level + objects (schemas, languages and casts) use the object name as + the key. Second-level (schema-owned) objects usually use the + schema name and the object name as the key. Some object types + need longer keys, e.g., operators need schema name, operator + symbols, left argument and right argument. + + Each class implementing an object type specifies a + :attr:`keylist` attribute, i.e., a list giving the names of + attributes making up the key. + """ + lst = [getattr(self, k) for k in self.keylist] + return len(lst) == 1 and lst[0] or tuple(lst) + + def identifier(self): + """Returns a full identifier for the database object + + :return: string + + This is used by :meth:`comment`, :meth:`alter_owner` and + :meth:`drop` to generate SQL syntax referring to the object. + It does not include the object type, but it may include (in + overridden methods) other elements, e.g., the arguments to a + function. + """ + return quote_id(self.__dict__[self.keylist[0]]) + + def to_map(self, db, no_owner=False, no_privs=False, deepcopy=True): + """Convert an object to a YAML-suitable format + + :param db: db used to tie the objects together + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + + The return value, a Python dictionary, is equivalent to a YAML + or JSON object. + """ + import copy + if deepcopy: + dct = copy.deepcopy(self.__dict__) + else: + dct = self.__dict__.copy() + for key in self.keylist: + del dct[key] + if self.description is None: + del dct['description'] + if no_owner or self.owner is None: + del dct['owner'] + if len(self.privileges) == 0 or no_privs: + del dct['privileges'] + else: + dct['privileges'] = self.map_privs() + + # Never dump the oid + dct.pop('oid', None) + + # Only dump dependencies that can't be inferred from the context + deps = set(dct.pop('depends_on', ())) + deps -= self.get_implied_deps(db) + if deps: + dct['depends_on'] = sorted([dep.extern_key() for dep in deps]) + + # Drop any private attributes + for k in list(dct.keys()): + if k.startswith('_'): + del dct[k] + + return dct + + def map_privs(self): + """Return a list of access privileges on the current object + + :return: list + """ + privlist = [] + for prv in self.privileges: + if prv: + privlist.append(privileges_to_map(prv, self.allprivs, + self.owner)) + sorted_privlist = [] + for sortedItem in sorted([list(i.keys())[0] for i in privlist]): + sorted_privlist.append([item for item in privlist + if list(item.keys())[0] == sortedItem][0]) + return sorted_privlist + + def set_oldname(self, inobj): + """Set oldname attribute if present in the input YAML map + + :param inobj: YAML map of input object + """ + if 'oldname' in inobj: + self.oldname = inobj.get('oldname') + + def fix_privileges(self): + """Adjust raw privilege information from YAML map""" + if len(self.privileges) > 0: + if self.owner is None: + raise ValueError( + "%s '%s' has privileges but no owner information" % ( + self.objtype.capitalize(), self.name)) + else: + self.privileges = privileges_from_map( + self.privileges, self.allprivs, self.owner) + + def _comment_text(self): + """Return the text for the SQL COMMENT statement + + :return: string + """ + if self.description is not None: + return "'%s'" % self.description.replace("'", "''") + else: + return 'NULL' + + def comment(self): + """Return SQL statement to create a COMMENT on the object + + :return: SQL statement + """ + return "COMMENT ON %s %s IS %s" % ( + self.objtype, self.identifier(), self._comment_text()) + + def alter_owner(self, owner=None): + """Return ALTER statement to set the OWNER of an object + + :return: SQL statement + """ + if self.owner != owner: + return "ALTER %s %s OWNER TO %s" % ( + self.objtype, self.identifier(), owner or self.owner) + else: + return [] + + def rename(self, oldname): + """Return SQL statement to RENAME the object + + :param oldname: the old name for the object + :return: SQL statement + """ + return "ALTER %s %s RENAME TO %s" % ( + self.objtype, quote_id(oldname), quote_id(self.name)) + + def create(self, dbversion=None): + raise NotImplementedError + + def create_sql(self, dbversion=None): + if hasattr(self, 'oldname') and self.oldname is not None: + return self.rename(self.oldname) + else: + return self.create(dbversion) + + def alter(self, inobj, no_owner=False): + """Generate SQL to transform an existing database object + + :param inobj: a YAML map defining the new object + :return: list of SQL statements + + Compares the current object to an input object and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if not no_owner and self.owner is not None and inobj.owner is not None: + if inobj.owner != self.owner: + stmts.append(self.alter_owner(inobj.owner)) + stmts.append(self.diff_privileges(inobj)) + stmts.append(self.diff_description(inobj)) + return stmts + + def drop(self): + """Generate SQL to drop the current object + + :return: list of SQL statements + """ + return ["DROP %s %s" % (self.objtype, self.identifier())] + + def diff_privileges(self, inobj): + """Generate SQL statements to grant or revoke privileges + + :param inobj: a YAML map defining the input object + :return: list of SQL statements + """ + return diff_privs(self, self.privileges, inobj, inobj.privileges) + + def diff_description(self, inobj): + """Generate SQL statements to add or change COMMENTs + + :param inobj: a YAML map defining the input object + :return: list of SQL statements + """ + stmts = [] + if self.description is not None: + if inobj.description is not None: + if self.description != inobj.description: + self.description = inobj.description + stmts.append(self.comment()) + else: + self.description = None + stmts.append(self.comment()) + else: + if inobj.description is not None: + self.description = inobj.description + stmts.append(self.comment()) + return stmts + + def get_deps(self, db): + """Return all the objects the object depends on + + The base implementation returns the explicit dependencies. Subclasses + may extend this to include implicit ones, which are implied e.g. by + containment in the yaml (such as an object on the schema they are on, a + constraint on the domain it is defined for. + + :return: set of `DbObject` + """ + deps = set() + + # The explicit dependencies + for dep in self.depends_on: + if isinstance(dep, str): + dep = db._get_by_extkey(dep) + deps.add(dep) + + for dep in self.get_implied_deps(db): + deps.add(dep) + + return deps + + def get_implied_deps(self, db): + """Return the dependencies the object can handle without being explicit + + :return: set of `DbObject` + """ + return set() + + +class DbSchemaObject(DbObject): + "A database object that is owned by a certain schema" + + def __init__(self, name, schema='public', description=None, **attrs): + super(DbSchemaObject, self).__init__(name, description, **attrs) + self.schema = schema + + def identifier(self): + """Return a full identifier for a schema object + + :return: string + """ + return "%s.%s" % (quote_id(self.schema), quote_id(self.name)) + + def qualname(self, schema=None, objname=None): + """Return the schema-qualified name of self or a related object + + :return: string + """ + if self.schema == schema and self.name == objname: + return self.identifier() + if objname is None: + objname = self.name + return "%s.%s" % (quote_id(schema or self.schema), quote_id(objname)) + + def unqualify(self, objname): + """Adjust the object name if it is qualified + + :param objname: object name + :return: unqualified object name + """ + if '.' in objname: + (sch, objname) = split_schema_obj(objname, self.schema) + assert sch == self.schema + return objname + else: + return objname + + def extern_filename(self, ext='yaml'): + """Return a filename to be used to output external files + + :param ext: file extension + :return: filename string + """ + return super(DbSchemaObject, self).extern_filename(ext, True) + + def rename(self, oldname): + """Return SQL statement to RENAME the schema object + + :param oldname: the old name for the schema object + :return: SQL statement + """ + return "ALTER %s %s.%s RENAME TO %s" % ( + self.objtype, quote_id(self.schema), quote_id(oldname), + quote_id(self.name)) + + def get_implied_deps(self, db): + deps = super(DbSchemaObject, self).get_implied_deps(db) + + # The schema of the object (if any) is always a dependency + if hasattr(self, 'schema'): + s = db.schemas.get(self.schema) + if s: + deps.add(s) + + return deps + + +class DbObjectDict(dict): + """A dictionary of database objects, all of the same type. + + However, note that "type" sometimes refers to a polymorphic class. + For example, a :class:`ConstraintDict` holds objects of type + :class:`Constraint`, but the actual objects may be of class + :class:`CheckConstraint`, :class:`PrimaryKey`, etc. + """ + + cls = DbObject + """The possibly-polymorphic class, derived from :class:`DbObject` that + the objects belong to. + """ + + def __init__(self, dbconn=None): + """Initialize the dictionary + + :param dbconn: a DbConnection object + + If dbconn is not None, the _from_catalog method is called to + initialize the dictionary from the catalogs. + """ + dict.__init__(self) + self.by_oid = {} + self.dbconn = dbconn + if dbconn: + self._from_catalog() + + def _from_catalog(self): + """Initialize the dictionary by querying the catalogs + + This is may be overridden by derived classes as needed. + """ + for obj in self.fetch(): + if hasattr(obj, 'options'): + if type(obj.options) is list: + obj.options = sorted(obj.options) + self[obj.key()] = obj + if hasattr(obj, 'oid'): + self.by_oid[obj.oid] = obj + + def to_map(self, db, opts): + """Convert the object dictionary to a regular dictionary + + :param db: db used to tie the objects together + :param opts: options to include/exclude information, etc. + :return: dictionary + + Invokes the `to_map` method of each object to construct the + dictionary. If `opts` specifies a directory, the objects are + written to files in that directory. + """ + objdict = {} + for objkey in sorted(self.keys()): + obj = self[objkey] + objmap = obj.to_map(db, opts.no_owner, opts.no_privs) + if objmap is not None: + extkey = obj.extern_key() + outobj = {extkey: objmap} + if opts.multiple_files: + filepath = obj.extern_filename() + with open(os.path.join(opts.metadata_dir, filepath), + 'a') as f: + f.write(yamldump(outobj)) + outobj = {extkey: filepath} + objdict.update(outobj) + return objdict + + def fetch(self): + """Fetch all objects from the catalogs using the associated + :meth:`query` methods. + + :return: list of self.cls (polymorphic) objects + + """ + self.query = self.cls.query(self.dbconn.version) + data = self.dbconn.fetchall(self.query) + self.dbconn.rollback() + return [self.cls(**dict(row)) for row in data] diff --git a/pyrseas/dbobject/cast.py b/pyrseas/dbobject/cast.py new file mode 100644 index 0000000..1565f60 --- /dev/null +++ b/pyrseas/dbobject/cast.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.cast + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Cast derived from DbObject and + CastDict derived from DbObjectDict. +""" +from . import DbObject, DbObjectDict, commentable +from . import split_func_args + + +CONTEXTS = {'a': 'assignment', 'e': 'explicit', 'i': 'implicit'} +METHODS = {'f': 'function', 'i': 'inout', 'b': 'binary coercible'} + + +class Cast(DbObject): + """A cast from a source type to a target type""" + + keylist = ['source', 'target'] + single_extern_file = True + catalog = 'pg_cast' + + def __init__(self, source, target, description, function, context, method, + oid=None): + """Initialize the cast + + :param source: source data type (from castsource) + :param target: target data type (from casttarget) + :param description: comment text (from obj_description()) + :param function: function to perform the cast (from castfunc) + :param context: context indicator (from castcontext) + :param method: method indicator (from castmethod) + """ + super(Cast, self).__init__('%s AS %s' % (source, target), description) + self._init_own_privs(None, []) + self.source = source + self.target = target + self.function = function + if len(context) == 1: + self.context = CONTEXTS[context] + else: + self.context = context + if len(method) == 1: + self.method = METHODS[method] + else: + self.method = method + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT castsource::regtype AS source, + casttarget::regtype AS target, + CASE WHEN castmethod = 'f' THEN castfunc::regprocedure + ELSE NULL::regprocedure END AS function, + castcontext AS context, castmethod AS method, + obj_description(c.oid, 'pg_cast') AS description, c.oid + FROM pg_cast c + JOIN pg_type s ON (castsource = s.oid) + JOIN pg_namespace sn ON (s.typnamespace = sn.oid) + JOIN pg_type t ON (casttarget = t.oid) + JOIN pg_namespace tn ON (t.typnamespace = tn.oid) + LEFT JOIN pg_proc p ON (castfunc = p.oid) + LEFT JOIN pg_namespace pn ON (p.pronamespace = pn.oid) + WHERE substring(sn.nspname for 3) != 'pg_' + OR substring(tn.nspname for 3) != 'pg_' + OR (castfunc != 0 AND substring(pn.nspname for 3) != 'pg_') + ORDER BY castsource, casttarget""" + + @staticmethod + def from_map(source, target, inobj): + """Initialize a Cast instance from a YAML map + + :param source: source type name + :param target: target type name + :param inobj: YAML map of the cast + :return: cast instance + """ + return Cast( + source, target, inobj.pop('description', None), + inobj.pop('function', None), inobj.get('context'), + inobj.get('method')) + + def extern_key(self): + """Return the key to be used in external maps for this cast + + :return: string + """ + return '%s (%s as %s)' % (self.objtype.lower(), self.source, + self.target) + + def identifier(self): + """Return a full identifier for a cast object + + :return: string + """ + return "(%s AS %s)" % (self.source, self.target) + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert a cast to a YAML-suitable format + + :return: dictionary + """ + dct = super(Cast, self).to_map(db) + del dct['name'] + if self.function is None: + del dct['function'] + return dct + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the cast + + :return: SQL statements + """ + with_clause = "\n WITH" + if self.function is not None: + with_clause += " FUNCTION %s" % self.function + elif self.method == 'inout': + with_clause += " INOUT" + else: + with_clause += "OUT FUNCTION" + as_clause = '' + if self.context == 'assignment': + as_clause = "\n AS ASSIGNMENT" + elif self.context == 'implicit': + as_clause = "\n AS IMPLICIT" + return ["CREATE CAST (%s AS %s)%s%s" % ( + self.source, self.target, with_clause, as_clause)] + + def get_implied_deps(self, db): + deps = super(Cast, self).get_implied_deps(db) + + # Types may be not found because they can be builtins + source = db.find_type(self.source) + if source: + deps.add(source) + + target = db.find_type(self.target) + if target: + deps.add(target) + + # The function instead we expect it exists + if self.method == 'function': + f = db.functions.find(*split_func_args(self.function)) + if f is not None: + deps.add(f) + + return deps + + +class CastDict(DbObjectDict): + "The collection of casts in a database" + + cls = Cast + + def from_map(self, incasts, newdb): + """Initialize the dictionary of casts by converting the input map + + :param incasts: YAML map defining the casts + :param newdb: collection of dictionaries defining the database + """ + for key in incasts: + if not key.startswith('cast (') or ' AS ' not in key.upper() \ + or key[-1:] != ')': + raise KeyError("Unrecognized object type: %s" % key) + asloc = key.upper().find(' AS ') + src = key[6:asloc] + trg = key[asloc + 4:-1] + inobj = incasts[key] + if not inobj: + raise ValueError("Cast '%s' has no specification" % key[5:]) + self[(src, trg)] = Cast.from_map(src, trg, inobj) diff --git a/pyrseas/dbobject/collation.py b/pyrseas/dbobject/collation.py new file mode 100644 index 0000000..c07c17b --- /dev/null +++ b/pyrseas/dbobject/collation.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.collation + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Collation and CollationDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable + + +class Collation(DbSchemaObject): + """A collation definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_collation' + + def __init__(self, name, schema, description, owner, lc_collate, lc_ctype, + oid=None): + """Initialize the collation + + :param name: collation name (from collname) + :param description: comment text (from obj_description()) + :param schema: schema name (from colnamespace) + :param owner: owner name (from rolname via collowner) + :param lc_collate: LC_COLLATE (from collcollate) + :param lc_ctype: LC_CTYPE (from collctype) + """ + super(Collation, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.lc_collate = lc_collate + self.lc_ctype = lc_ctype + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, collname AS name, rolname AS owner, + collcollate AS lc_collate, collctype AS lc_ctype, + obj_description(c.oid, 'pg_collation') AS description, c.oid + FROM pg_collation c + JOIN pg_roles r ON (r.oid = collowner) + JOIN pg_namespace n ON (collnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, collname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Collation instance from a YAML map + + :param name: collation name + :param name: schema map + :param inobj: YAML map of the collation + :return: Collation instance + """ + obj = Collation( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('lc_collate', None), + inobj.pop('lc_ctype', None)) + obj.set_oldname(inobj) + return obj + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the collation + + :return: SQL statements + """ + return ["CREATE COLLATION %s (\n LC_COLLATE = '%s'," + "\n LC_CTYPE = '%s')" % ( + self.qualname(), self.lc_collate, self.lc_ctype)] + + +class CollationDict(DbObjectDict): + "The collection of collations in a database." + + cls = Collation + + def from_map(self, schema, inmap): + """Initialize the dictionary of collations by examining the input map + + :param schema: the schema owing the collations + :param inmap: the input YAML map defining the collations + """ + for key in inmap: + if not key.startswith('collation '): + raise KeyError("Unrecognized object type: %s" % key) + name = key[10:] + inobj = inmap[key] + self[(schema.name, name)] = Collation.from_map(name, schema, inobj) diff --git a/pyrseas/dbobject/column.py b/pyrseas/dbobject/column.py new file mode 100644 index 0000000..e004fd8 --- /dev/null +++ b/pyrseas/dbobject/column.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.column + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Column derived from + DbSchemaObject and ColumnDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject, quote_id +from .privileges import privileges_from_map, add_grant, diff_privs + + +IDENTITY_TYPES = {'a': 'always', 'd': 'by default'} + + +class Column(DbSchemaObject): + "A table column or attribute of a composite type" + + keylist = ['schema', 'table'] # plus attribute number + allprivs = 'arwx' + + def __init__(self, name, schema, table, number, type, description=None, + privileges=[], not_null=True, default=None, identity=None, + collation=None, statistics=None, inherited=False, + dropped=False): + """Initialize the column + + :param name: column/attribute name (from attname) + :param schema: schema name (from nspname attrelid/relnamespace) + :param table: table/composite type name (from relame via attrelid) + :param description: comment text (from obj_description()) + :param privileges: access privileges (from attacl) + :param number: attribute number (from attnum) + :param type: data type (from atttypid/atttypmod) + :param not_null: not null constraint (from attnotnull) + :param default: default value expression (from pg_attrdef.adbin) + :param identity: type of identity column (from attidentity) + :param collation: collation name (from collname via attcollation) + :param statistics: statistics detail level (from attstattarget) + :param inherited: inherited indicator (from attinhcount) + :param dropped: dropped indicator (from attisdropped) + """ + super(Column, self).__init__(name, schema, description) + self._init_own_privs(None, privileges) + self.table = table + self.number = number + self.type = type + self.not_null = not_null + self.default = default + if identity == '' or identity is None: + self.identity = None + elif identity is not None and len(identity) == 1: + self.identity = IDENTITY_TYPES[identity] + else: + self.identity = identity + assert self.identity is None or \ + self.identity in IDENTITY_TYPES.values() + self.collation = collation + self.statistics = statistics + self.inherited = inherited + self.dropped = dropped + self._table = None + self._type = None + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS table, attname AS name, + attnum AS number, format_type(atttypid, atttypmod) AS type, + attnotnull AS not_null, attinhcount > 0 AS inherited, + pg_get_expr(adbin, adrelid) AS default, + attidentity AS identity, attstattarget AS statistics, + collname AS collation, attisdropped AS dropped, + array_to_string(attacl, ',') AS privileges, + col_description(c.oid, attnum) AS description + FROM pg_attribute JOIN pg_class c ON (attrelid = c.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + LEFT JOIN pg_attrdef ON (attrelid = pg_attrdef.adrelid + AND attnum = pg_attrdef.adnum) + LEFT JOIN pg_collation l ON (attcollation = l.oid) + WHERE relkind in ('c', 'r', 'f', 'p', 'v', 'm') + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND attnum > 0 + ORDER BY nspname, relname, attnum""" + + @staticmethod + def from_map(name, table, num, inobj): + """Initialize a Column instance from a YAML map + + :param name: column name + :param table: table map + :param num: column number + :param inobj: YAML map of the column + :return: Column instance + """ + obj = Column( + name, table.schema, table.name, num, inobj.pop('type', None), + inobj.pop('description', None), inobj.pop('privileges', []), + inobj.pop('not_null', False), inobj.pop('default', None), + inobj.pop('identity', None), inobj.pop('collation', None), + inobj.pop('statistics', None), inobj.pop('inherited', False), + inobj.pop('dropped', False)) + obj.set_oldname(inobj) + if len(obj.privileges) > 0: + if table.owner is None: + raise ValueError("Column '%s.%s' has privileges but " + "no owner information" % (table.name, name)) + obj.privileges = privileges_from_map( + obj.privileges, obj.allprivs, table.owner) + return obj + + def to_map(self, db, no_privs): + """Convert a column to a YAML-suitable format + + :param no_privs: exclude privilege information + :return: dictionary + """ + if self.dropped: + return None + dct = super(Column, self).to_map(db, False, no_privs, deepcopy=False) + del dct['number'], dct['name'], dct['dropped'] + if not self.not_null: + dct.pop('not_null') + if self.default is None: + dct.pop('default') + if self.identity is None: + dct.pop('identity') + if self.collation is None or self.collation == 'default': + dct.pop('collation') + if not self.inherited: + dct.pop('inherited') + if self.statistics is None or self.statistics == -1: + dct.pop('statistics') + return {self.name: dct} + + def add(self): + """Return a string to specify the column in a CREATE or ALTER TABLE + + :return: partial SQL statement + """ + stmt = "%s %s" % (quote_id(self.name), self.type) + if self.not_null: + stmt += ' NOT NULL' + if self.default is not None: + stmt += ' DEFAULT ' + self.default + if self.identity is not None: + stmt += " GENERATED %s AS IDENTITY" % self.identity.upper() + stmt += " (%s)" % self._owner_seq.add_inline() + if self.collation is not None and self.collation != 'default': + stmt += ' COLLATE "%s"' % self.collation + return (stmt, '' if self.description is None else self.comment()) + + def add_privs(self): + """Generate SQL statements to grant privileges on new column + + :return: list of SQL statements + """ + return [add_grant(self._table, priv, self.name) + for priv in self.privileges] + + def diff_privileges(self, incol): + """Generate SQL statements to grant or revoke privileges + + :param incol: a YAML map defining the input column + :return: list of SQL statements + """ + return [diff_privs(self._table, self.privileges, incol._table, + incol.privileges, self.name)] + + def comment(self): + """Return a SQL COMMENT statement for the column + + :return: SQL statement + """ + return "COMMENT ON COLUMN %s.%s IS %s" % ( + self._table.qualname(), self.name, self._comment_text()) + + def drop(self): + """Return string to drop the column via ALTER TABLE + + :return: SQL statement + """ + if self.dropped: + return [] + if self._table is not None: + (comptype, objtype) = (self._table.objtype, 'COLUMN') + compname = self._table.qualname() + elif self._type is not None: + (comptype, objtype) = ('TYPE', 'ATTRIBUTE') + compname = self._type.qualname() + else: + raise TypeError("Cannot determine type of %s", self.name) + return "ALTER %s %s DROP %s %s" % (comptype, compname, objtype, + quote_id(self.name)) + + def rename(self, newname): + """Return SQL statement to RENAME the column + + :param newname: the new name of the object + :return: SQL statement + """ + if self._table is not None: + (comptype, objtype) = (self._table.objtype, 'COLUMN') + compname = self._table.qualname() + elif self._type is not None: + (comptype, objtype) = ('TYPE', 'ATTRIBUTE') + compname = self._type.qualname() + else: + raise TypeError("Cannot determine type of %s", self.name) + stmt = "ALTER %s %s RENAME %s %s TO %s" % ( + comptype, compname, objtype, self.name, newname) + self.name = newname + return stmt + + def alter(self, incol): + """Generate SQL to transform an existing column + + :param insequence: a YAML map defining the new column + :return: list of partial SQL statements + + Compares the column to an input column and generates partial + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + base = "ALTER COLUMN %s " % quote_id(self.name) + # check NOT NULL + if not self.not_null and incol.not_null: + stmts.append(base + "SET NOT NULL") + if self.not_null and not incol.not_null: + stmts.append(base + "DROP NOT NULL") + # check data types + if self.type is None: + raise ValueError("Column '%s' missing datatype" % self.name) + if incol.type is None: + raise ValueError("Input column '%s' missing datatype" % incol.name) + if self.type != incol.type: + # validate type conversion? + stmts.append(base + "TYPE %s" % incol.type) + # check DEFAULTs + if self.default is None and incol.default is not None: + stmts.append(base + "SET DEFAULT %s" % incol.default) + if self.default is not None: + if incol.default is None: + stmts.append(base + "DROP DEFAULT") + elif self.default != incol.default: + stmts.append(base + "SET DEFAULT %s" % incol.default) + # check STATISTICS + if self.statistics is not None: + if self.statistics == -1 and (incol.statistics is not None + and incol.statistics != -1): + stmts.append(base + "SET STATISTICS %d" % incol.statistics) + if self.statistics != -1 and (incol.statistics is None + or incol.statistics == -1): + stmts.append(base + "SET STATISTICS -1") + + return (", ".join(stmts), self.diff_description(incol)) + + +class ColumnDict(DbObjectDict): + "The collection of columns in tables in a database" + + cls = Column + + def _from_catalog(self): + """Initialize the dictionary of columns by querying the catalogs""" + for col in self.fetch(): + sch, tbl = col.key() + if (sch, tbl) not in self: + self[(sch, tbl)] = [] + self[(sch, tbl)].append(col) + + def from_map(self, table, incols): + """Initialize the dictionary of columns by converting the input list + + :param table: table or type owning the columns/attributes + :param incols: YAML list defining the columns + """ + if not incols: + raise ValueError("Table '%s' has no columns" % table.name) + cols = self[(table.schema, table.name)] = [] + + for (num, incol) in enumerate(incols): + for key in incol: + if isinstance(incol[key], dict): + inobj = incol[key] + else: + inobj = {'type': incol[key]} + cols.append(Column.from_map(key, table, num, inobj)) diff --git a/pyrseas/dbobject/constraint.py b/pyrseas/dbobject/constraint.py new file mode 100644 index 0000000..12f932c --- /dev/null +++ b/pyrseas/dbobject/constraint.py @@ -0,0 +1,782 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.constraint + ~~~~~~~~~~~~~~~~~~ + + This module defines six classes: Constraint derived from + DbSchemaObject, CheckConstraint, PrimaryKey, ForeignKey and + UniqueConstraint derived from Constraint, and ConstraintDict + derived from DbObjectDict. + + TODO: UniqueConstraint and PrimaryKey are nearly identical. + Perhaps the latter should inherit from the former. +""" +import copy + +from . import DbObjectDict, DbSchemaObject +from . import quote_id, split_schema_obj, commentable +from .index import Index + + +class Constraint(DbSchemaObject): + """A constraint definition, such as a primary key, foreign key or + unique constraint. This also covers check constraints on domains.""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_constraint' + + def __init__(self, name, schema, table, description): + """Initialize the constraint + + :param name: constraint name (from conname) + :param schema: schema name (from connamespace) + :param table: table/domain name (from conrelid/contypid) + :param description: comment text (from obj_description()) + """ + super(Constraint, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = self.unqualify(table) + + def key_columns(self): + """Return comma-separated list of key column names + + :return: string + """ + return ", ".join([quote_id(col) for col in self.columns]) + + def _normalize_columns(self): + "Replace integer column numbers by column names" + if isinstance(self.columns[0], int): + self.columns = [self._table.columns[k - 1].name + for k in self.columns] + + def create(self, dbversion=None): + # TODO: is add really needed? + return self.add() + + @commentable + def add(self): + """Return string to add the constraint via ALTER TABLE + + :return: SQL statement + + Works as is for primary keys and unique constraints but has + to be overridden for check constraints and foreign keys. + """ + stmts = [] + tblspc = '' + if self.tablespace is not None: + tblspc = " USING INDEX TABLESPACE %s" % self.tablespace + stmts.append("ALTER TABLE %s ADD CONSTRAINT %s %s (%s)%s" % ( + self._table.qualname(), quote_id(self.name), + self.objtype, self.key_columns(), tblspc)) + if self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + return stmts + + def drop(self): + """Return string to drop the constraint via ALTER TABLE + + :return: SQL statement + """ + return ["ALTER %s %s DROP CONSTRAINT %s" % ( + self._table.objtype, self._table.qualname(), quote_id(self.name))] + + def comment(self): + """Return SQL statement to create COMMENT on constraint + + :return: SQL statement + """ + return "COMMENT ON CONSTRAINT %s ON %s IS %s" % ( + quote_id(self.name), self._table.qualname(), self._comment_text()) + + def get_implied_deps(self, db): + from .table import Table + from .dbtype import Domain + deps = super(Constraint, self).get_implied_deps(db) + + if isinstance(self._table, Table): + deps.add(db.tables[self.schema, self.table]) + elif isinstance(self._table, Domain): + deps.add(db.types[self.schema, self.table]) + else: + raise KeyError("Constraint '%s.%s' on unknown type/class" % ( + self.schema, self.name)) + + return deps + + +class CheckConstraint(Constraint): + "A check constraint definition" + + def __init__(self, name, schema, table, description, columns, + expression, is_domain_check=False, inherited=False, + oid=None): + """Initialize the check constraint + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param expression: constraint expression (from consrc) + :param is_domain_check: is constraint for a domain? (from contypid) + :param inherited: is it inherited? (from coninhcount) + """ + super(CheckConstraint, self).__init__(name, schema, table, description) + self.columns = columns + if expression[0] == '(': + assert expression[-1] == ')' + self.expression = expression + self.is_domain_check = is_domain_check + self.inherited = inherited + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + CASE WHEN contypid = 0 THEN conrelid::regclass::text + ELSE contypid::regtype::text END AS table, + contypid != 0 AS is_domain_check, conkey AS columns, + pg_get_expr(conbin, conrelid) AS expression, + coninhcount > 0 AS inherited, c.oid, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'c' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, target, inobj): + """Initialize a CheckConstraint instance from a YAML map + + :param name: constraint name + :param table: table map + :param target: column (default) or domain indicator + :param inobj: YAML map of the constraint + :return: CheckConstraint instance + """ + obj = CheckConstraint( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('expression', None), + (target != ''), inobj.pop('inherited', False)) + if 'depends_on' in inobj: + obj.depends_on.extend(inobj.pop('depends_on')) + return obj + + @property + def objtype(self): + return "CHECK" + + def to_map(self, db, dbcols): + """Convert a check constraint definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + dct = super(CheckConstraint, self).to_map(db, deepcopy=False) + dct.pop('is_domain_check') + if not self.inherited: + dct.pop('inherited') + if dbcols is not None and self.columns is not None: + dct['columns'] = [dbcols[k - 1] for k in self.columns] + else: + dct.pop('columns') + return {self.name: copy.deepcopy(dct)} + + @commentable + def add(self): + """Return string to add the CHECK constraint via ALTER TABLE + + :return: SQL statement + """ + # Don't generate inherited constraints + if self.inherited: + return [] + + if self.expression[0] != '(': + expr = "(%s)" % self.expression + else: + expr = self.expression + return ["ALTER %s %s ADD CONSTRAINT %s %s %s" % ( + self._table.objtype, self._table.qualname(), quote_id(self.name), + self.objtype, expr)] + + def drop(self): + if self.inherited: + return [] + else: + return super(CheckConstraint, self).drop() + + def alter(self, inchk): + """Generate SQL to transform an existing CHECK constraint + + :param inchk: a YAML map defining the new CHECK constraint + :return: list of SQL statements + + Compares the CHECK constraint to an input constraint and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + if inchk.expression != self.expression: + if inchk.expression.lower() != self.expression.lower(): + stmts.append(self.drop()) + stmts.append(inchk.add()) + stmts.append(self.diff_description(inchk)) + return stmts + + +class PrimaryKey(Constraint): + "A primary key constraint definition" + + def __init__(self, name, schema, table, description, columns, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the primary key + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is PK inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(PrimaryKey, self).__init__(name, schema, table, description) + self.columns = columns + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + amname AS access_method, spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl on (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'p' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a PrimaryKey instance from a YAML map + + :param name: key name + :param table: table map + :param inobj: YAML map of the primary key + :return: PrimaryKey instance + """ + return PrimaryKey( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + + @property + def objtype(self): + return "PRIMARY KEY" + + def to_map(self, db, dbcols): + """Convert a primary key definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + dct = super(PrimaryKey, self).to_map(db, deepcopy=False) + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.tablespace is None: + dct.pop('tablespace') + if '_table' in dct: + del dct['_table'] + dct['columns'] = [dbcols[k - 1] for k in self.columns] + return {self.name: copy.deepcopy(dct)} + + def alter(self, inpk): + """Generate SQL to transform an existing primary key + + :param inpk: a YAML map defining the new primary key + :return: list of SQL statements + + Compares the primary key to an input primary key and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + self._normalize_columns() + if inpk.columns != self.columns: + stmts.append(self.drop()) + stmts.append(inpk.add()) + if inpk.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self._table.qualname()) + stmts.append(self.diff_description(inpk)) + return stmts + + +ACTIONS = {'r': 'restrict', 'c': 'cascade', 'n': 'set null', + 'd': 'set default'} +MATCH_TYPES = {'f': 'full', 'p': 'partial', 's': 'simple'} + + +class ForeignKey(Constraint): + "A foreign key constraint definition" + + def __init__(self, name, schema, table, description, columns, + ref_table, ref_cols, on_update, on_delete, match, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the foreign key + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param ref_table: referenced table (from confrelid) + :param ref_cols: referenced columns (from confkey) + :param on_update: update action code (from confupdtype) + :param on_delete: delete action code (from confdeltype) + :param match: match action code (from confmatchtype) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is PK inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(ForeignKey, self).__init__(name, schema, table, description) + self.columns = columns + (self.ref_schema, self.ref_table) = split_schema_obj(ref_table, schema) + self.ref_cols = ref_cols + if on_update is not None and len(on_update) == 1: + self.on_update = None if on_update == 'a' else ACTIONS[on_update] + else: + assert on_update is None or on_update in ACTIONS.values() + self.on_update = on_update + if on_delete is not None and len(on_delete) == 1: + self.on_delete = None if on_delete == 'a' else ACTIONS[on_delete] + else: + assert on_delete is None or on_delete in ACTIONS.values() + self.on_delete = on_delete + if match is not None and len(match) == 1: + self.match = MATCH_TYPES[match] + else: + assert match is None or match in MATCH_TYPES.values() + self.match = 'simple' if match is None else match + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + confrelid::regclass AS ref_table, confkey AS ref_cols, + confupdtype AS on_update, confdeltype AS on_delete, + confmatchtype AS match, amname AS access_method, + spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl ON (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'f' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a ForeignKey instance from a YAML map + + :param name: key name + :param table: table map + :param inobj: YAML map of the foreign key + :return: ForeignKey instance + """ + if 'references' not in inobj: + raise KeyError("Constraint '%s' missing references" % name) + refs = inobj['references'] + if 'table' not in refs: + raise KeyError("Constraint '%s' missing table reference" % name) + ref_table = refs['table'] + if 'schema' in refs and refs['schema'] != table.schema: + ref_table = "%s.%s" % (refs['schema'], ref_table) + if 'columns' not in refs: + raise KeyError("Constraint '%s' missing reference columns" % name) + obj = ForeignKey( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), ref_table, refs['columns'], + inobj.pop('on_update', None), inobj.pop('on_delete', None), + inobj.pop('match', None), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + obj.depends_on.extend(inobj.get('depends_on', ())) + return obj + + @property + def objtype(self): + return "FOREIGN KEY" + + def _normalize_columns(self): + "Replace integer column numbers by column names" + super(ForeignKey, self)._normalize_columns() + if isinstance(self.ref_cols[0], int): + self.ref_cols = [self._references.columns[k - 1].name + for k in self.ref_cols] + + def ref_columns(self): + """Return comma-separated list of reference column names + + :return: string + """ + return ", ".join(self.ref_cols) + + def to_map(self, db, dbcols, refcols): + """Convert a foreign key definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + self._normalize_columns() + dct = super(ForeignKey, self).to_map(db, deepcopy=False) + if '_table' in dct: + del dct['_table'] + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + for attr in ('tablespace', 'on_update', 'on_delete'): + if getattr(self, attr) is None: + dct.pop(attr) + if self.match == 'simple': + dct.pop('match') + dct['references'] = {'table': dct['ref_table'], + 'columns': self.ref_cols} + if 'ref_schema' in dct: + dct['references'].update(schema=self.ref_schema) + dct.pop('ref_schema') + dct.pop('ref_table') + dct.pop('ref_cols') + + return {self.name: copy.deepcopy(dct)} + + @commentable + def add(self): + """Return string to add the foreign key via ALTER TABLE + + :return: SQL statement + """ + match = '' + if self.match is not None and self.match != 'simple': + match = " MATCH %s" % self.match.upper() + actions = '' + if self.on_update is not None: + actions = " ON UPDATE %s" % self.on_update.upper() + if self.on_delete is not None: + actions += " ON DELETE %s" % self.on_delete.upper() + if self.deferrable: + actions += " DEFERRABLE" + if self.deferred: + actions += " INITIALLY DEFERRED" + + return "ALTER TABLE %s ADD CONSTRAINT %s FOREIGN KEY (%s) " \ + "REFERENCES %s (%s)%s%s" % ( + self._table.qualname(), quote_id(self.name), + self.key_columns(), self._references.qualname(), + self.ref_columns(), match, actions) + + def alter(self, infk): + """Generate SQL to transform an existing foreign key + + :param infk: a YAML map defining the new foreign key + :return: list of SQL statements + + Compares the foreign key to an input foreign key and generates + SQL statements to transform it into the one represented by the + input. + """ + stmts = [] + self._normalize_columns() + if infk.columns != self.columns or infk.ref_cols != self.ref_cols \ + or infk.match != self.match or infk.on_update != self.on_update \ + or infk.on_delete != self.on_delete: + stmts.append(self.drop()) + stmts.append(infk.add()) + stmts.append(self.diff_description(infk)) + return stmts + + def get_implied_deps(self, db): + deps = super(ForeignKey, self).get_implied_deps(db) + + # add the table we reference + deps.add(self._references) + + # A fkey needs a pkey, unique constraint or complete unique index + # defined on the fields it references to be restored. + idx = self._find_referenced_index(db, self._references) + if idx: + deps.add(idx) + + return deps + + def _find_referenced_index(self, db, ref_table): + pkey = ref_table.primary_key + if pkey is not None and pkey.columns == self.ref_cols: + return pkey + + for uc in list(ref_table.unique_constraints.values()): + uc._normalize_columns() + if uc.columns == self.ref_cols: + return uc + + if hasattr(ref_table, 'indexes'): + if isinstance(self.ref_cols[0], int): + col_names = [ref_table.columns[i-1].name + for i in self.ref_cols] + else: + col_names = self.ref_cols + + for idx in list(ref_table.indexes.values()): + if getattr(idx, 'unique', False) \ + and not getattr(idx, 'predicate', None) \ + and idx.keys == col_names: + return idx + + +class UniqueConstraint(Constraint): + "A unique constraint definition" + + def __init__(self, name, schema, table, description, columns, + access_method='btree', tablespace=None, cluster=False, + inherited=False, deferrable=False, deferred=False, + oid=None): + """Initialize the unique constraint + + :param name-description: see Constraint.__init__ params + :param columns: list of columns (should only be one) (from conkey) + :param access_method: index access method (from am_name via conindid) + :param tablespace: storage tablespace (from spcname) + :param cluster: is index clustered? (from indisclustered) + :param inherited: is it inherited? (from coninhcount) + :param deferrable: is constraint deferrable? (from condeferrable) + :param deferred: is constraint deferred? (from condeferred) + """ + super(UniqueConstraint, self).__init__( + name, schema, table, description) + self.columns = columns + self.access_method = access_method + self.tablespace = tablespace + self.cluster = cluster + self.inherited = inherited + self.deferrable = deferrable + self.deferred = deferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return r""" + SELECT conname AS name, nspname AS schema, + conrelid::regclass AS table, conkey AS columns, + condeferrable AS deferrable, condeferred AS deferred, + amname AS access_method, spcname AS tablespace, c.oid, + indisclustered AS cluster, coninhcount > 0 AS inherited, + obj_description(c.oid, 'pg_constraint') AS description + FROM pg_constraint c + JOIN pg_namespace ON (connamespace = pg_namespace.oid) + JOIN pg_index i ON (indexrelid = conindid) + JOIN pg_class cl on (indexrelid = cl.oid) + JOIN pg_am on (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (cl.reltablespace = t.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND contype = 'u' + AND contypid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_type'::regclass) + AND conrelid NOT IN (SELECT objid FROM pg_depend + WHERE deptype = 'e' AND classid = 'pg_class'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a UniqueConstraint instance from a YAML map + + :param name: constraint name + :param table: table map + :param inobj: YAML map of the constraint + :return: UniqueConstraint instance + """ + return UniqueConstraint( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('columns', []), inobj.pop('access_method', 'btree'), + inobj.pop('tablespace', None), inobj.pop('cluster', False), + inobj.pop('inherited', False), inobj.pop('deferrable', False), + inobj.pop('deferred', False)) + + @property + def objtype(self): + return "UNIQUE" + + def to_map(self, db, dbcols): + """Convert a unique constraint definition to a YAML-suitable format + + :param dbcols: dictionary of dbobject columns + :return: dictionary + """ + self._normalize_columns() + dct = super(UniqueConstraint, self).to_map(db, deepcopy=False) + if self.access_method == 'btree': + dct.pop('access_method') + for attr in ('inherited', 'deferrable', 'deferred', 'cluster'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.tablespace is None: + dct.pop('tablespace') + return {self.name: copy.deepcopy(dct)} + + def alter(self, inuc): + """Generate SQL to transform an existing unique constraint + + :param inuc: a YAML map defining the new unique constraint + :return: list of SQL statements + + Compares the unique constraint to an input unique constraint + and generates SQL statements to transform it into the one + represented by the input. + """ + stmts = [] + self._normalize_columns() + if inuc.columns != self.columns: + stmts.append(self.drop()) + stmts.append(inuc.add()) + if inuc.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self._table.qualname(), quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self._table.qualname()) + stmts.append(self.diff_description(inuc)) + return stmts + + +MATCHTYPES_PRE93 = {'f': 'full', 'p': 'partial', 'u': 'simple'} + + +class ConstraintDict(DbObjectDict): + "The collection of table or column constraints in a database" + + cls = Constraint + + def _from_catalog(self): + """Initialize the dictionary of constraints by querying the catalogs""" + for cls in (CheckConstraint, PrimaryKey, ForeignKey, + UniqueConstraint): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, table, inconstrs, target=''): + """Initialize the dictionary of constraints by converting the input map + + :param table: table affected by the constraints + :param inconstrs: YAML map defining the constraints + :param target: column or domain indicator + """ + if 'check_constraints' in inconstrs: + chks = inconstrs['check_constraints'] + for cns in chks: + inobj = chks[cns] + self[(table.schema, table.name, cns)] = \ + CheckConstraint.from_map(cns, table, target, inobj) + if 'primary_key' in inconstrs: + cns = list(inconstrs['primary_key'].keys())[0] + inobj = inconstrs['primary_key'][cns] + self[(table.schema, table.name, cns)] = PrimaryKey.from_map( + cns, table, inobj) + if 'foreign_keys' in inconstrs: + fkeys = inconstrs['foreign_keys'] + for cns in fkeys: + inobj = fkeys[cns] + self[(table.schema, table.name, cns)] = ForeignKey.from_map( + cns, table, inobj) + if 'unique_constraints' in inconstrs: + uconstrs = inconstrs['unique_constraints'] + for cns in uconstrs: + inobj = uconstrs[cns] + self[(table.schema, table.name, cns)] = \ + UniqueConstraint.from_map(cns, table, inobj) + + def link_refs(self, db): + for c in list(self.values()): + if isinstance(c, ForeignKey): + # The constraint depends on an index. Which one is accidental: + # it depends e.g. on which suitable index was available when + # the constraint was defined. So here we drop the dependency + # on the introspected one, while in get_implied_deps we give + # our best shot to suggest one to depend on. This way we don't + # need expliciting the dependency in yaml. + c.depends_on = [obj for obj in c.depends_on + if not isinstance(obj, Index)] + + if isinstance(c, (PrimaryKey, UniqueConstraint)): + # The constraint creates implicitly an index, so it depends on + # any extra dependencies the index has. These may include e.g. + # an operator class for a non-builtin type. + idx = db.indexes.get((c.schema, c.table, c.name)) + if idx: + c.depends_on.extend(idx.depends_on) diff --git a/pyrseas/dbobject/conversion.py b/pyrseas/dbobject/conversion.py new file mode 100644 index 0000000..32b1ae8 --- /dev/null +++ b/pyrseas/dbobject/conversion.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.conversion + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Conversion and ConversionDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable + + +class Conversion(DbSchemaObject): + """A conversion definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_conversion' + + def __init__(self, name, schema, description, owner, source_encoding, + dest_encoding, function, default=False, + oid=None): + """Initialize the conversion + + :param name: conversion name (from conname) + :param schema: schema name (from connamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via conowner) + :param source_encoding: source encoding (from conforencoding) + :param source_encoding: destination encoding (from contoencoding) + :param function: conversion function (from conproc) + :param default: indicates this is default conversion (from condefault) + """ + super(Conversion, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.source_encoding = source_encoding + self.dest_encoding = dest_encoding + self.function = function + self.default = default + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, conname AS name, rolname AS owner, + pg_encoding_to_char(c.conforencoding) AS source_encoding, + pg_encoding_to_char(c.contoencoding) AS dest_encoding, + conproc AS function, condefault AS default, c.oid, + obj_description(c.oid, 'pg_conversion') AS description + FROM pg_conversion c + JOIN pg_roles r ON (r.oid = conowner) + JOIN pg_namespace n ON (connamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, conname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Conversion instance from a YAML map + + :param name: conversion name + :param table: schema map + :param inobj: YAML map of the conversion + :return: Conversion instance + """ + obj = Conversion( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('source_encoding'), + inobj.pop('dest_encoding'), inobj.pop('function'), + inobj.pop('default', False)) + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert a conversion to a YAML-suitable format + + :return: dictionary + """ + dct = super(Conversion, self).to_map(db, no_owner) + if not self.default: + del dct['default'] + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the conversion + + :return: SQL statements + """ + dflt = '' + if self.default: + dflt = 'DEFAULT ' + return ["CREATE %sCONVERSION %s\n FOR '%s' TO '%s' FROM %s" % ( + dflt, self.qualname(), self.source_encoding, + self.dest_encoding, self.function)] + + +class ConversionDict(DbObjectDict): + "The collection of conversions in a database." + + cls = Conversion + + def from_map(self, schema, inmap): + """Initialize the dictionary of conversions by examining the input map + + :param schema: the schema owing the conversions + :param inmap: the input YAML map defining the conversions + """ + for key in inmap: + if not key.startswith('conversion '): + raise KeyError("Unrecognized object type: %s" % key) + cnv = key[11:] + inobj = inmap[key] + self[(schema.name, cnv)] = Conversion.from_map(cnv, schema, inobj) diff --git a/pyrseas/dbobject/dbtype.py b/pyrseas/dbobject/dbtype.py new file mode 100644 index 0000000..3420c46 --- /dev/null +++ b/pyrseas/dbobject/dbtype.py @@ -0,0 +1,740 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.dbtype + ~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines seven classes: DbType derived from + DbSchemaObject, BaseType, Composite, Domain, Enum and Range derived + from DbType, and TypeDict derived from DbObjectDict. +""" + +from . import DbObjectDict, DbSchemaObject +from . import split_schema_obj, commentable, ownable +from .constraint import CheckConstraint + +ALIGNMENT_TYPES = {'c': 'char', 's': 'int2', 'i': 'int4', 'd': 'double'} +STORAGE_TYPES = {'p': 'plain', 'e': 'external', 'm': 'main', 'x': 'extended'} + + +class DbType(DbSchemaObject): + """A user-defined type, such as a composite, domain or enum""" + + keylist = ['schema', 'name'] + catalog = 'pg_type' + + def __init__(self, name, schema, description, owner, privileges): + """Initialize the type + + :param name: type name (from typname) + :param schema: schema name (from typnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via typowner) + :param privileges: access privileges (from typacl) + """ + super(DbType, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + + @property + def objtype(self): + return "TYPE" + + def find_defining_funcs(self, dbfuncs): + return [] + + +class BaseType(DbType): + """A base type""" + + def __init__(self, name, schema, description, owner, privileges, + input, output, receive=None, send=None, typmod_in=None, + typmod_out=None, analyze=None, internallength=1, + alignment=None, storage=None, delimiter=',', + category=None, preferred=False, + oid=None): + """Initialize the base type + + :param name-privileges: see DbType.__init__ params + :param input: input function (see typinput) + :param output: output function (see typoutput) + :param receive: input conversion function (see typreceive) + :param send: output conversion function (see typsend) + :param typmod_in: type modifier input function (see typmodin) + :param typmod_out: type modifier output function (see typmodout) + :param analyze: custom ANALYZE function (see typanalyze) + :param internallength: length in bytes or -1 (variable) (see typlen) + :param alignment: storage alignment (see typalign) + :param storage: storage type for varlena types (see typstorage) + :param delimiter: delimiter character for array type (see typdelim) + :param category: PG data type classification (see typcategory) + :param preferred: preferred cast target? (see typispreferred) + """ + super(BaseType, self).__init__(name, schema, description, owner, + privileges) + self.input = self.unqualify(input) + self.output = self.unqualify(output) + self.receive = receive if receive != '-' else None + self.send = send if send != '-' else None + self.typmod_in = typmod_in if typmod_in != '-' else None + self.typmod_out = typmod_out if typmod_out != '-' else None + self.analyze = analyze if analyze != '-' else None + self.internallength = internallength + if alignment is not None and len(alignment) == 1: + self.alignment = ALIGNMENT_TYPES[alignment] + else: + assert alignment in ALIGNMENT_TYPES.values() + self.alignment = alignment + if storage is not None and len(storage) == 1: + self.storage = STORAGE_TYPES[storage] + else: + assert storage in STORAGE_TYPES.values() + self.storage = storage + self.delimiter = delimiter + self.category = category + self.preferred = preferred + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + typinput::regproc AS input, typoutput::regproc AS output, + typreceive::regproc AS receive, typsend::regproc AS send, + typmodin::regproc AS typmod_in, + typmodout::regproc AS typmod_out, + typanalyze::regproc AS analyze, + typlen AS internallength, typalign AS alignment, + typstorage AS storage, typdelim AS delimiter, + typcategory AS category, typispreferred AS preferred, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'b' AND typarray != 0 + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a BaseType instance from a YAML map + + :param name: BaseType name + :param schema: schema map + :param inobj: YAML map of the BaseType + :return: BaseType instance + """ + obj = BaseType( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('input', None), inobj.pop('output', None), + inobj.pop('receive', None), inobj.pop('send', None), + inobj.pop('typmod_in', None), inobj.pop('typmod_out', None), + inobj.pop('analyze', None), inobj.pop('internallength', 1), + inobj.pop('alignment', None), inobj.pop('storage', None), + inobj.pop('delimiter', ','), inobj.pop('category', None), + inobj.pop('preferred', False)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + dct = super(BaseType, self).to_map(db, no_owner, no_privs) + for attr in ('receive', 'send', 'typmod_in', 'typmod_out', 'analyze', + 'alignment', 'storage', 'category'): + if getattr(self, attr) is None: + dct.pop(attr) + if self.internallength < 0: + dct['internallength'] = 'variable' + if self.delimiter == ',': + dct.pop('delimiter') + if self.preferred is False: + dct.pop('preferred') + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the base type + + :return: SQL statements + """ + stmts = [] + opt_clauses = [] + if self.send is not None: + opt_clauses.append("SEND = %s" % self.send) + if self.receive is not None: + opt_clauses.append("RECEIVE = %s" % self.receive) + opt_clauses.append("INTERNALLENGTH = %s" % self.internallength) + if self.alignment is not None: + opt_clauses.append("ALIGNMENT = %s" % self.alignment) + if self.storage is not None: + opt_clauses.append("STORAGE = %s" % self.storage) + if self.delimiter is not None and self.delimiter != ',': + opt_clauses.append("DELIMITER = '%s'" % self.delimiter) + if self.category is not None: + opt_clauses.append("CATEGORY = '%s'" % self.category) + if self.preferred: + opt_clauses.append("PREFERRED = TRUE") + stmts.append("CREATE TYPE %s (\n INPUT = %s," + "\n OUTPUT = %s%s%s)" % ( + self.qualname(), self.input, self.output, + opt_clauses and ',\n ' or '', + ',\n '.join(opt_clauses))) + return stmts + + def get_implied_deps(self, db): + deps = super(BaseType, self).get_implied_deps(db) + for f in self.find_defining_funcs(db.functions): + deps.add(f) + + return deps + + def find_defining_funcs(self, dbfuncs): + rv = [] + for attr, arg in [ + ('input', 'cstring'), ('output', self.qualname()), + ('receive', 'internal'), ('send', self.qualname())]: + f = getattr(self, attr, None) + if not f: + continue + fschema, fname = split_schema_obj(f, self.schema) + rv.append(dbfuncs[fschema, fname, arg]) + return rv + + def drop(self): + """Generate SQL to drop the type (and related functions) + + :return: list of SQL statements + """ + # The CASCADE clause is required to also drop the related + # functions. There is a cyclic dependency so the dependency + # graph cannot be used. The functions will not be explicitly + # dropped. + return ["DROP %s %s CASCADE" % (self.objtype, self.identifier())] + + +class Composite(DbType): + """A composite type""" + + def __init__(self, name, schema, description, owner, privileges, + oid=None): + """Initialize the composite type + + :param name-privileges: see DbType.__init__ params + """ + super(Composite, self).__init__(name, schema, description, owner, + privileges) + self.attributes = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND (typtype = 'c' AND relkind = 'c') + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Composite instance from a YAML map + + :param name: Composite name + :param schema: schema map + :param inobj: YAML map of the Composite + :return: Composite instance + """ + obj = Composite( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + if len(self.attributes) == 0: + return + dct = super(Composite, self).to_map(db, no_owner, no_privs) + attrs = [] + for attr in self.attributes: + att = attr.to_map(db, False) + if att: + attrs.append(att) + dct['attributes'] = attrs + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the composite type + + :return: SQL statements + """ + attrs = [] + for att in self.attributes: + attrs.append(" " + att.add()[0]) + return ["CREATE TYPE %s AS (\n%s)" % ( + self.qualname(), ",\n".join(attrs))] + + def alter(self, intype): + """Generate SQL to transform an existing composite type + + :param intype: the new composite type + :return: list of SQL statements + + Compares the type to an input type and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if len(intype.attributes) == 0: + raise KeyError("Composite '%s' has no attributes" % intype.name) + attrnames = [attr.name for attr in self.attributes if not attr.dropped] + dbattrs = len(attrnames) + + base = "ALTER TYPE %s\n " % (self.qualname()) + # check input attributes + for (num, inattr) in enumerate(intype.attributes): + if hasattr(inattr, 'oldname'): + assert(self.attributes[num].name == inattr.oldname) + stmts.append(self.attributes[num].rename(inattr.name)) + # check existing attributes + if num < dbattrs and self.attributes[num].name == inattr.name: + (stmt, descr) = self.attributes[num].alter(inattr) + if stmt: + stmts.append(base + stmt) + if descr: + stmts.append(descr) + # add new attributes + elif inattr.name not in attrnames: + (stmt, descr) = inattr.add() + stmts.append(base + "ADD ATTRIBUTE %s" % stmt) + if descr: + stmts.append(descr) + + # Check the columns to drop + inattrnames = set(attr.name for attr in intype.attributes) + for attr in self.attributes: + if attr.name not in inattrnames: + stmts.append(attr.drop()) + + stmts.append(super(Composite, self).alter(intype)) + + return stmts + + def get_implied_deps(self, db): + deps = super(Composite, self).get_implied_deps(db) + for col in self.attributes: + type = db.find_type(col.type) + if type is not None: + deps.add(type) + + return deps + + +class Enum(DbType): + "An enumerated type definition" + + def __init__(self, name, schema, description, owner, privileges, + labels, oid=None): + """Initialize the enumerated type + + :param name-privileges: see DbType.__init__ params + """ + super(Enum, self).__init__(name, schema, description, owner, + privileges) + self.labels = labels + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + array_to_string(typacl, ',') AS privileges, + ARRAY(SELECT enumlabel FROM pg_enum e + WHERE t.oid = enumtypid + ORDER BY e.oid) AS labels, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'e' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize an Enum instance from a YAML map + + :param name: Enum name + :param schema: schema map + :param inobj: YAML map of the Enum + :return: Enum instance + """ + obj = Enum( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('labels', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the enum + + :return: SQL statements + """ + lbls = ["'%s'" % lbl for lbl in self.labels] + return ["CREATE TYPE %s AS ENUM (%s)" % ( + self.qualname(), ",\n ".join(lbls))] + + def alter(self, intype, no_owner=False): + """Generate SQL to transform an existing enum type + + :param intype: the new enum type + :return: list of SQL statements + + Compares the enum to an input enum and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if self.labels != intype.labels: + stmts.append(self.drop()) + stmts.append(intype.create()) + stmts.append(super(Enum, self).alter(intype, no_owner)) + return stmts + + +class Domain(DbType): + "A domain definition" + + def __init__(self, name, schema, description, owner, privileges, + type, not_null=False, default=None, + oid=None): + """Initialize the domain + + :param name-privileges: see DbType.__init__ params + :param type: type modifier (see typtypmod) + :param not_null: not null indicator (see typnotnull) + :param default: default value (see typdefault) + """ + super(Domain, self).__init__(name, schema, description, owner, + privileges) + self.type = type + self.not_null = not_null + self.default = default + self.check_constraints = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, typname AS name, rolname AS owner, + format_type(typbasetype, typtypmod) AS type, + typnotnull AS not_null, typdefault AS default, + array_to_string(typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_roles r ON (r.oid = typowner) + JOIN pg_namespace n ON (typnamespace = n.oid) + LEFT JOIN pg_class c ON (typrelid = c.oid) + WHERE typisdefined AND typtype = 'd' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize an Domain instance from a YAML map + + :param name: Domain name + :param schema: schema map + :param inobj: YAML map of the Domain + :return: Domain instance + """ + obj = Domain( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('type', None), inobj.pop('not_null', False), + inobj.pop('default', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "DOMAIN" + + def to_map(self, db, no_owner, no_privs): + """Convert a domain to a YAML-suitable format + + :param no_owner: exclude domain owner information + :return: dictionary + """ + dct = super(Domain, self).to_map(db, no_owner, no_privs) + if self.not_null is False: + dct.pop('not_null') + if self.default is None: + dct.pop('default') + if len(self.check_constraints) > 0: + for cns in list(self.check_constraints.values()): + dct['check_constraints'].update( + self.check_constraints[cns.name].to_map(db, None)) + else: + dct.pop('check_constraints') + + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the domain + + :return: SQL statements + """ + create = "CREATE DOMAIN %s AS %s" % (self.qualname(), self.type) + if self.not_null: + create += ' NOT NULL' + if self.default is not None: + create += ' DEFAULT ' + str(self.default) + return [create] + + def get_implied_deps(self, db): + deps = super(Domain, self).get_implied_deps(db) + + # depend on the base type + # don't give errors in case it's a builtin + tschema, tname = split_schema_obj(self.type) + type = db.types.get((tschema, tname)) + if type: + deps.add(type) + + # In my testing database there is a dependency on the output + # function of the base type. TODO: investigate more. + if hasattr(type, 'output'): + fschema, fname = split_schema_obj(type.output) + func = db.functions[fschema, fname, type.qualname()] + deps.add(func) + + return deps + + +class Range(DbType): + "A range type definition" + + def __init__(self, name, schema, description, owner, privileges, + subtype, canonical=None, subtype_diff=None, + oid=None): + """Initialize the range type + + :param name-privileges: see DbType.__init__ params + :param subtype: type of range elements (from rngsubtype) + """ + super(Range, self).__init__(name, schema, description, owner, + privileges) + self.subtype = subtype + self.canonical = canonical if canonical != '-' else None + self.subtype_diff = subtype_diff if subtype_diff != '-' else None + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, t.typname AS name, rolname AS owner, + st.typname AS subtype, rn.rngcanonical AS canonical, + rn.rngsubdiff AS subtype_diff, + array_to_string(t.typacl, ',') AS privileges, + obj_description(t.oid, 'pg_type') AS description, t.oid + FROM pg_type t JOIN pg_range rn ON rngtypid = t.oid + JOIN pg_type st ON rngsubtype = st.oid + JOIN pg_roles r ON (r.oid = t.typowner) + JOIN pg_namespace n ON (t.typnamespace = n.oid) + WHERE t.typisdefined AND t.typtype = 'r' + AND nspname NOT IN ('pg_catalog', 'pg_toast', + 'information_schema') + AND t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_type'::regclass) + ORDER BY nspname, t.typname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a Range instance from a YAML map + + :param name: Range name + :param schema: schema map + :param inobj: YAML map of the Range + :return: Range instance + """ + obj = Range( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('subtype', None), inobj.pop('canonical', None), + inobj.pop('subtype_diff', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a range type to a YAML-suitable format + + :param no_owner: exclude type owner information + :return: dictionary + """ + dct = super(Range, self).to_map(db, no_owner, no_privs) + for attr in ('canonical', 'subtype_diff'): + if getattr(self, attr) is None: + dct.pop(attr) + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the range + + :return: SQL statements + """ + clauses = [] + if self.canonical is not None: + clauses.append("CANONICAL = %s" % self.canonical) + if self.subtype_diff is not None: + clauses.append("SUBTYPE_DIFF = %s" % self.subtype_diff) + return ["CREATE TYPE %s AS RANGE (SUBTYPE = %s%s%s)" % ( + self.qualname(), self.subtype, + clauses and ",\n " or "", ",\n ".join(clauses))] + + def alter(self, intype, no_owner=False): + """Generate SQL to transform an existing range type + + :param intype: the new range type + :return: list of SQL statements + + Compares the range to an input range and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + stmts.append(super(Range, self).alter(intype, no_owner)) + return stmts + + +class TypeDict(DbObjectDict): + "The collection of user-defined types in a database" + + cls = DbType + # TODO: consider to fetch all the objects belonging to extensions: + # not to dump them but to trace dependency from objects to the extension + + def _from_catalog(self): + """Initialize the dictionary of types by querying the catalogs""" + for cls in (BaseType, Composite, Domain, Enum, Range): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of types by converting the input map + + :param schema: schema owning the types + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + for k in inobjs: + (objtype, spc, key) = k.partition(' ') + if spc != ' ' or objtype not in ['domain', 'type']: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'domain': + inobj = inobjs[k] + self[(schema.name, key)] = obj = Domain.from_map( + key, schema, inobj) + newdb.constraints.from_map(obj, inobj, 'd') + elif objtype == 'type': + inobj = inobjs[k] + if 'input' in inobj: + self[(schema.name, key)] = BaseType.from_map( + key, schema, inobj) + elif 'attributes' in inobj: + self[(schema.name, key)] = obj = Composite.from_map( + key, schema, inobj) + try: + newdb.columns.from_map(obj, inobj['attributes']) + except KeyError as exc: + exc.args = ("Type '%s' has no attributes" % key, ) + raise + elif 'labels' in inobj: + self[(schema.name, key)] = obj = Enum.from_map( + key, schema, inobj) + elif 'subtype' in inobj: + self[(schema.name, key)] = obj = Range.from_map( + key, schema, inobj) + else: + raise KeyError("Unrecognized object type: %s" % k) + + def find(self, obj): + """Find a type given its name. + + The name can contain modifiers such as arrays '[]' and attributes '(3)' + + Return None if not found. + """ + schema, name = split_schema_obj(obj) + name = name.rstrip('[](,)0123456789') + return self.get((schema, name)) + + def link_refs(self, dbcolumns, dbconstrs, dbfuncs): + """Connect various objects to their corresponding types or domains + + :param dbcolumns: dictionary of columns + :param dbconstrs: dictionary of constraints + :param dbfuncs: dictionary of functions + + Fills the `check_constraints` dictionaries for each domain by + traversing the `dbconstrs` dictionary. Fills the attributes + list for composite types. Fills the dependent functions + dictionary for base types. + """ + for (sch, typ) in dbcolumns: + if (sch, typ) in self: + assert isinstance(self[(sch, typ)], Composite) + self[(sch, typ)].attributes = dbcolumns[(sch, typ)] + for attr in dbcolumns[(sch, typ)]: + attr._type = self[(sch, typ)] + for (sch, typ, cns) in dbconstrs: + constr = dbconstrs[(sch, typ, cns)] + if isinstance(constr, CheckConstraint) and \ + constr.is_domain_check: + constr._table = dbtype = self[(sch, typ)] + dbtype.check_constraints.update({cns: constr}) diff --git a/pyrseas/dbobject/eventtrig.py b/pyrseas/dbobject/eventtrig.py new file mode 100644 index 0000000..c52fece --- /dev/null +++ b/pyrseas/dbobject/eventtrig.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.eventtrig + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: EventTrigger derived from + DbObject, and EventTriggerDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable +from .function import split_schema_func, join_schema_func + +EXEC_PROC = 'EXECUTE PROCEDURE ' + +ENABLE_MODES = {'O': True, 'D': False, 'R': 'replica', 'A': 'always'} + + +class EventTrigger(DbObject): + """An event trigger""" + + keylist = ['name'] + catalog = 'pg_event_trigger' + + def __init__(self, name, description, owner, event, procedure, + enabled=False, tags=None, + oid=None): + """Initialize the event trigger + + :param name: trigger name (from evtname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via evtowner) + :param event: event that causes firing (from evtevent) + :param procedure: function to be called (from evtfoid) + :param enabled: replication mode firing control (from evtenabled) + :param tags: command tags (from evttags) + """ + super(EventTrigger, self).__init__(name, description) + self._init_own_privs(owner, []) + self.event = event + if procedure[-2:] == '()': + procedure = procedure[:-2] + self.procedure = split_schema_func(None, procedure) + if enabled is False or enabled is True: + self.enabled = enabled + elif len(enabled) == 1: + self.enabled = ENABLE_MODES[enabled] + else: + assert enabled is not None and enabled in ENABLE_MODES.values() + self.enabled = enabled + self.tags = tags + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT evtname AS name, evtevent AS event, rolname AS owner, + evtenabled AS enabled, evtfoid::regprocedure AS procedure, + evttags AS tags, t.oid, + obj_description(t.oid, 'pg_event_trigger') AS description + FROM pg_event_trigger t JOIN pg_roles ON (evtowner = pg_roles.oid) + WHERE t.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e') + ORDER BY name""" + + @staticmethod + def from_map(name, inobj): + """Initialize an event trigger instance from a YAML map + + :param name: trigger name + :param inobj: YAML map of the event trigger + :return: event trigger instance + """ + obj = EventTrigger( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('event', None), inobj.pop('procedure', None), + inobj.pop('enabled', False), inobj.pop('tags', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "EVENT TRIGGER" + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert an event trigger definition to a YAML-suitable format + + :param db: db used to tie the objects together + :return: dictionary + """ + dct = super(EventTrigger, self).to_map(db, no_owner) + dct['procedure'] = join_schema_func(self.procedure) + "()" + if self.tags is None: + dct.pop('tags') + return dct + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the event trigger + + :return: SQL statements + """ + filter = '' + if self.tags is not None: + filter = "\n WHEN tag IN (%s)" % ", ".join( + ["'%s'" % tag for tag in self.tags]) + return ["CREATE %s %s\n ON %s%s\n EXECUTE PROCEDURE %s()" % ( + self.objtype, quote_id(self.name), self.event, filter, + join_schema_func(self.procedure))] + + def get_implied_deps(self, db): + deps = super(EventTrigger, self).get_implied_deps(db) + sch, fnc = self.procedure + deps.add(db.functions[(sch, fnc, '')]) + return deps + + +class EventTriggerDict(DbObjectDict): + "The collection of event triggers in a database" + + cls = EventTrigger + + def _from_catalog(self): + """Initialize the dictionary of triggers by querying the catalogs""" + if self.dbconn.version < 90300: + return + super(EventTriggerDict, self)._from_catalog() + + def from_map(self, intriggers, newdb): + """Initialize the dictionary of triggers by converting the input map + + :param intriggers: YAML map defining the event triggers + :param newdb: dictionary of input database + """ + for key in intriggers: + if not key.startswith('event trigger '): + raise KeyError("Unrecognized object type: %s" % key) + trg = key[14:] + inobj = intriggers[key] + if not inobj: + raise ValueError("Event trigger '%s' has no specification" % + trg) + self[trg] = EventTrigger.from_map(trg, inobj) diff --git a/pyrseas/dbobject/extension.py b/pyrseas/dbobject/extension.py new file mode 100644 index 0000000..8c4733f --- /dev/null +++ b/pyrseas/dbobject/extension.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.extension + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Extension derived from DbObject, + and ExtensionDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable + + +class Extension(DbObject): + """An extension""" + + keylist = ['name'] + single_extern_file = True + catalog = 'pg_extension' + + def __init__(self, name, description, owner, schema, version=None, + oid=None): + """Initialize the extension + + :param name: extension name (from extlname) + :param description: comment text (from obj_description()) + :param schema: schema name (from extnamespace) + :param owner: owner name (from rolname via extowner) + :param version: version name (from extversion) + """ + super(Extension, self).__init__(name, description) + self._init_own_privs(owner, []) + self.schema = schema + self.version = version + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT e.extname AS name, n.nspname AS schema, e.extversion AS version, + r.rolname AS owner, + obj_description(e.oid, 'pg_extension') AS description, e.oid + FROM pg_extension e + JOIN pg_roles r ON (r.oid = e.extowner) + JOIN pg_namespace n ON (e.extnamespace = n.oid) + WHERE n.nspname != 'information_schema' + ORDER BY e.extname""" + + @staticmethod + def from_map(name, inobj): + """Initialize an Extension instance from a YAML map + + :param name: extension name + :param inobj: YAML map of the extension + :return: extension instance + """ + return Extension( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.get('schema'), inobj.pop('version', None)) + + def get_implied_deps(self, db): + """Return the implied dependencies of the object + + :param db: the database where this object exists + :return: set of `DbObject` + """ + deps = super(Extension, self).get_implied_deps(db) + if self.schema is not None: + s = db.schemas.get(self.schema) + if s: + deps.add(s) + + return deps + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the extension + + :return: SQL statements + """ + opt_clauses = [] + if self.schema is not None and self.schema not in ( + 'pg_catalog', 'public'): + opt_clauses.append("SCHEMA %s" % quote_id(self.schema)) + if self.version is not None: + opt_clauses.append("VERSION '%s'" % self.version) + return ["CREATE EXTENSION %s%s" % ( + quote_id(self.name), ('\n ' + '\n '.join(opt_clauses)) + if opt_clauses else '')] + + def alter(self, inobj, no_owner=True): + """Generate SQL to transform an existing extension + + :param inobj: a YAML map defining the new extension + :return: list of SQL statements + + This exists because ALTER EXTENSION does not permit altering + the owner. + """ + return super(Extension, self).alter(inobj, no_owner=no_owner) + +CORE_LANGS = [ + "plpgsql", + "pltcl", + "pltclu", + "plperl", + "plperlu", + "plpythonu", + "plpython2u", + "plpython3u"] + +class ExtensionDict(DbObjectDict): + "The collection of extensions in a database" + + cls = Extension + + def _from_catalog(self): + """Initialize the dictionary of extensions by querying the catalogs""" + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, inexts, newdb): + """Initialize the dictionary of extensions by converting the input map + + :param inexts: YAML map defining the extensions + :param newdb: dictionary of input database + """ + for key in inexts: + if not key.startswith('extension '): + raise KeyError("Unrecognized object type: %s" % key) + name = key[10:] + inobj = inexts[key] + self[name] = Extension.from_map(name, inobj) + if self[name].name in CORE_LANGS: + lang = {'language %s' % self[name].name: { + '_ext': 'e', 'owner': self[name].owner}} + newdb.languages.from_map(lang) diff --git a/pyrseas/dbobject/foreign.py b/pyrseas/dbobject/foreign.py new file mode 100644 index 0000000..0da995a --- /dev/null +++ b/pyrseas/dbobject/foreign.py @@ -0,0 +1,693 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.foreign + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines nine classes: DbObjectWithOptions derived from + DbObject, ForeignDataWrapper, ForeignServer and UserMapping + derived from DbObjectWithOptions, ForeignDataWrapperDict, + ForeignServerDict and UserMappingDict derived from DbObjectDict, + ForeignTable derived from DbObjectWithOptions and Table, and + ForeignTableDict derived from ClassDict. +""" +from . import DbObjectDict, DbObject +from . import quote_id, commentable, ownable, grantable +from .table import ClassDict, Table + + +class DbObjectWithOptions(DbObject): + """Helper class for database objects with OPTIONS clauses""" + + def __init__(self, name, options): + """Initialize the database object with options + + :param name: object name (from fdwname, srvname, etc.) + :param options: object specific options (from fdwoptions, etc.) + """ + super(DbObjectWithOptions, self).__init__(name, None) + self.options = {} if options is None else options + + def to_map(self, db, no_owner=False, no_privs=False): + """Convert objects to a YAML-suitable format + + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(DbObjectWithOptions, self).to_map(db, no_owner, no_privs) + if len(self.options) == 0: + dct.pop('options') + return dct + + def options_clause(self): + """Create the OPTIONS clause + + :param optdict: the dictionary of options + :return: SQL OPTIONS clause + """ + opts = [] + for opt in self.options: + (nm, val) = opt.split('=', 1) + opts.append("%s '%s'" % (nm, val)) + return "OPTIONS (%s)" % ', '.join(opts) + + def diff_options(self, newopts): + """Compare options lists and generate SQL OPTIONS clause + + :newopts: list of new options + :return: SQL OPTIONS clause + + Generate ([ADD|SET|DROP key 'value') clauses from two lists in the + form of 'key=value' strings. + """ + def to_dict(optlist): + return dict(opt.split('=', 1) for opt in optlist) + + oldopts = {} + if len(self.options) > 0: + oldopts = to_dict(self.options) + newopts = to_dict(newopts) + clauses = [] + for key, val in list(newopts.items()): + if key not in oldopts: + clauses.append("%s '%s'" % (key, val)) + elif val != oldopts[key]: + clauses.append("SET %s '%s'" % (key, val)) + for key, val in list(oldopts.items()): + if key not in newopts: + clauses.append("DROP %s" % key) + return clauses and "OPTIONS (%s)" % ', '.join(clauses) or '' + + def alter(self, inobj): + """Generate SQL to transform an existing object with options + + :param inobj: a YAML map defining the new object + :return: list of SQL statements + """ + stmts = super(DbObjectWithOptions, self).alter(inobj) + newopts = [] + if len(inobj.options) > 0: + newopts = inobj.options + diff_opts = self.diff_options(newopts) + if diff_opts: + stmts.append("ALTER %s %s %s" % ( + self.objtype, self.identifier(), diff_opts)) + return stmts + + +class ForeignDataWrapper(DbObjectWithOptions): + """A foreign data wrapper definition""" + + single_extern_file = True + catalog = 'pg_foreign_data_wrapper' + + def __init__(self, name, options, description, owner, privileges, + handler=None, validator=None, + oid=None): + """Initialize the foreign data wrapper + + :param name-options: see DbObjectWithOptions.__init__ params + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via fdwowner) + :param privileges: access privileges (from fdwacl) + :param handler: handler function (from fdwhandler) + :param validator: validator function (from fdwvalidator) + """ + super(ForeignDataWrapper, self).__init__(name, options) + self.description = description + self._init_own_privs(owner, privileges) + self.handler = handler + self.validator = validator + self.servers = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS name, CASE WHEN fdwhandler = 0 THEN NULL + ELSE fdwhandler::regproc END AS handler, + CASE WHEN fdwvalidator = 0 THEN NULL + ELSE fdwvalidator::regproc END AS validator, + fdwoptions AS options, rolname AS owner, + array_to_string(fdwacl, ',') AS privileges, + obj_description(w.oid, 'pg_foreign_data_wrapper') AS + description, w.oid + FROM pg_foreign_data_wrapper w + JOIN pg_roles r ON (r.oid = fdwowner) + ORDER BY fdwname""" + + @staticmethod + def from_map(name, inobj): + """Initialize an FDW instance from a YAML map + + :param name: FDW name + :param inobj: YAML map of the FDW + :return: FDW instance + """ + obj = ForeignDataWrapper( + name, inobj.pop('options', {}), inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('handler', None), inobj.pop('validator', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "FOREIGN DATA WRAPPER" + + @property + def allprivs(self): + return 'U' + + def to_map(self, db, no_owner, no_privs): + """Convert wrappers and subsidiary objects to a YAML-suitable format + + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(ForeignDataWrapper, self).to_map(db, no_owner, no_privs) + for attr in ('handler', 'validator'): + if getattr(self, attr) is None: + dct.pop(attr) + srvs = {} + for srv in self.servers: + srvs.update(self.servers[srv].to_map(db, no_owner, no_privs)) + dct.update(srvs) + dct.pop('servers') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the data wrapper + + :return: SQL statements + """ + clauses = [] + for fnc in ('validator', 'handler'): + if getattr(self, fnc) is not None: + clauses.append("%s %s" % (fnc.upper(), getattr(self, fnc))) + if len(self.options) > 0: + clauses.append(self.options_clause()) + return ["CREATE FOREIGN DATA WRAPPER %s%s" % ( + quote_id(self.name), + clauses and '\n ' + ',\n '.join(clauses) or '')] + + +class ForeignDataWrapperDict(DbObjectDict): + "The collection of foreign data wrappers in a database" + + cls = ForeignDataWrapper + + def from_map(self, inwrappers, newdb): + """Initialize the dictionary of wrappers by examining the input map + + :param inwrappers: input YAML map defining the data wrappers + :param newdb: collection of dictionaries defining the database + """ + for key in inwrappers: + if not key.startswith('foreign data wrapper '): + raise KeyError("Unrecognized object type: %s" % key) + fdw = key[21:] + inobj = inwrappers[key] + inservs = {} + for key in inobj: + if key.startswith('server '): + inservs.update({key: inobj[key]}) + self[fdw] = ForeignDataWrapper.from_map(fdw, inobj) + newdb.servers.from_map(self[fdw], inservs, newdb) + + def link_refs(self, dbservers): + """Connect servers to their respective foreign data wrappers + + :param dbservers: dictionary of foreign servers + """ + for (fdw, srv) in dbservers: + dbserver = dbservers[(fdw, srv)] + assert self[fdw] + wrapper = self[fdw] + if not hasattr(wrapper, 'servers'): + wrapper.servers = {} + wrapper.servers.update({srv: dbserver}) + + +class ForeignServer(DbObjectWithOptions): + """A foreign server definition""" + + privobjtype = "FOREIGN SERVER" + keylist = ['wrapper', 'name'] + catalog = 'pg_foreign_server' + + def __init__(self, name, options, description, owner, privileges, + wrapper, type=None, version=None, + oid=None): + """Initialize the foreign server + + :param name-options: see DbObjectWithOptions.__init__ params + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via srvowner) + :param privileges: access privileges (from srvacl) + :param wrapper: foreign data wrapper (from fdwname via srvfdw) + :param type: server type (from srvtype) + :param version: version (from srvversion) + """ + super(ForeignServer, self).__init__(name, options) + self.description = description + self._init_own_privs(owner, privileges) + self.wrapper = wrapper + self.type = type + self.version = version + self.usermaps = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS wrapper, srvname AS name, srvtype AS type, + srvversion AS version, srvoptions AS options, + rolname AS owner, + array_to_string(srvacl, ',') AS privileges, s.oid, + obj_description(s.oid, 'pg_foreign_server') AS description + FROM pg_foreign_server s JOIN pg_roles r ON (r.oid = srvowner) + JOIN pg_foreign_data_wrapper w ON (srvfdw = w.oid) + ORDER BY fdwname, srvname""" + + @staticmethod + def from_map(name, wrapper, inobj): + """Initialize a foreign server instance from a YAML map + + :param name: server name + :param wrapper: FDW name + :param inobj: YAML map of the server + :return: foreign server instance + """ + obj = ForeignServer( + name, inobj.pop('options', {}), inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), wrapper, + inobj.pop('type', None), inobj.pop('version', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "SERVER" + + @property + def allprivs(self): + return 'U' + + def identifier(self): + """Returns a full identifier for the foreign server + + :return: string + """ + return quote_id(self.name) + + def to_map(self, db, no_owner, no_privs): + """Convert servers and subsidiary objects to a YAML-suitable format + + :param no_owner: exclude server owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(ForeignServer, self).to_map(db, no_owner, no_privs) + for attr in ('type', 'version'): + if getattr(self, attr) is None: + dct.pop(attr) + key = self.extern_key() + server = {key: dct} + server[key].pop('usermaps') + if len(self.usermaps) > 0: + umaps = {} + for umap in self.usermaps: + umaps.update({umap: self.usermaps[umap].to_map(db)}) + server[key]['user mappings'] = umaps + + return server + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the server + + :return: SQL statements + """ + clauses = [] + options = [] + for opt in ('type', 'version'): + if getattr(self, opt) is not None: + clauses.append("%s '%s'" % (opt.upper(), getattr(self, opt))) + if len(self.options) > 0: + options.append(self.options_clause()) + return ["CREATE SERVER %s%s\n FOREIGN DATA WRAPPER %s%s" % ( + quote_id(self.name), + clauses and ' ' + ' '.join(clauses) or '', + quote_id(self.wrapper), + options and '\n ' + ',\n '.join(options) or '')] + + def get_implied_deps(self, db): + deps = super(ForeignServer, self).get_implied_deps(db) + deps.add(db.fdwrappers[self.wrapper]) + return deps + + +class ForeignServerDict(DbObjectDict): + "The collection of foreign servers in a database" + + cls = ForeignServer + + def from_map(self, wrapper, inservers, newdb): + """Initialize the dictionary of servers by examining the input map + + :param wrapper: associated foreign data wrapper + :param inservers: input YAML map defining the foreign servers + :param newdb: collection of dictionaries defining the database + """ + for key in inservers: + if not key.startswith('server '): + raise KeyError("Unrecognized object type: %s" % key) + srv = key[7:] + inobj = inservers[key] + self[(wrapper.name, srv)] = serv = ForeignServer.from_map( + srv, wrapper.name, inobj) + if 'user mappings' in inobj: + newdb.usermaps.from_map(serv, inobj['user mappings']) + + def to_map(self, db, no_owner, no_privs): + """Convert the server dictionary to a regular dictionary + + :param no_owner: exclude server owner information + :param no_privs: exclude privilege information + :return: dictionary + + Invokes the `to_map` method of each server to construct a + dictionary of foreign servers. + """ + servers = {} + for srv in self: + servers.update(self[srv].to_map(db, no_owner, no_privs)) + return servers + + def link_refs(self, dbusermaps): + """Connect user mappings to their respective servers + + :param dbusermaps: dictionary of user mappings + """ + for (fdw, srv, usr) in dbusermaps: + dbusermap = dbusermaps[(fdw, srv, usr)] + assert self[(fdw, srv)] + server = self[(fdw, srv)] + server.usermaps.update({usr: dbusermap}) + + +class UserMapping(DbObjectWithOptions): + """A user mapping definition""" + + keylist = ['wrapper', 'server', 'name'] + catalog = 'pg_user_mappings' + + def __init__(self, name, options, wrapper, server, + oid=None): + """Initialize the user mapping + + :param name-options: see DbObjectWithOptions.__init__ params + :param wrapper: foreign data wrapper (from fdwname via srvfdw) + :param server: server name (from umserver) + :param version: version (from srvversion) + """ + super(UserMapping, self).__init__(name, options) + self.wrapper = wrapper + self.server = server + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT fdwname AS wrapper, s.srvname AS server, + CASE umuser WHEN 0 THEN 'PUBLIC' ELSE + usename END AS name, umoptions AS options, u.umid AS oid + FROM pg_user_mappings u + JOIN pg_foreign_server s ON (u.srvid = s.oid) + JOIN pg_foreign_data_wrapper w ON (srvfdw = w.oid) + ORDER BY fdwname, s.srvname, 3""" + + @staticmethod + def from_map(name, server, inobj): + """Initialize a user mapping instance from a YAML map + + :param name: mapping name + :param server: foreign server map + :param inobj: YAML map of the user mapping + :return: user mapping instance + """ + obj = UserMapping(name, inobj.pop('options', {}), server.wrapper, + server.name) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "USER MAPPING" + + def extern_key(self): + """Return the key to be used in external maps for this user mapping + + :return: string + """ + return self.name + + def identifier(self): + """Return a full identifier for a user mapping object + + :return: string + """ + return "FOR %s SERVER %s" % ( + self.name == 'PUBLIC' and 'PUBLIC' or quote_id(self.name), + quote_id(self.server)) + + def create(self, dbversion=None): + """Return SQL statements to CREATE the user mapping + + :return: SQL statements + """ + options = [] + if len(self.options) > 0: + options.append(self.options_clause()) + return ["CREATE USER MAPPING FOR %s\n SERVER %s%s" % ( + self.name == 'PUBLIC' and 'PUBLIC' or + quote_id(self.name), quote_id(self.server), + options and '\n ' + ',\n '.join(options) or '')] + + def get_implied_deps(self, db): + deps = super(UserMapping, self).get_implied_deps(db) + deps.add(db.fdwrappers[self.wrapper]) + return deps + + +class UserMappingDict(DbObjectDict): + "The collection of user mappings in a database" + + cls = UserMapping + + def from_map(self, server, inusermaps): + """Initialize the dictionary of mappings by examining the input map + + :param server: foreign server associated with mappings + :param inusermaps: input YAML map defining the user mappings + """ + for key in inusermaps: + inobj = inusermaps[key] + self[(server.wrapper, server.name, key)] = UserMapping.from_map( + key, server, inobj) + + def to_map(self, db): + """Convert the user mapping dictionary to a regular dictionary + + :return: dictionary + + Invokes the `to_map` method of each mapping to construct a + dictionary of user mappings. + """ + usermaps = {} + for um in self: + usermaps.update(self[um].to_map(db)) + return usermaps + + +class ForeignTable(Table, DbObjectWithOptions): + """A foreign table definition""" + + privobjtype = "TABLE" + catalog = 'pg_foreign_table' + + def __init__(self, name, schema, description, owner, privileges, + server=None, options={}, + oid=None): + """Initialize the foreign table + + :param name-privileges: see DbClass.__init__ params + :param server: foreign server (from ftserver) + :param options: table options (from ftoptions) + """ + super(ForeignTable, self).__init__(name, schema, description, owner, + privileges) + self.server = server + self.options = {} if options is None else options + self.columns = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, srvname AS server, + ftoptions AS options, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_foreign_table f ON (ftrelid = c.oid) + JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_foreign_server s ON (ftserver = s.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'f' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a foreign table instance from a YAML map + + :param name: foreign table name + :param name: schema name + :param inobj: YAML map of the table + :return: foreign table instance + """ + obj = ForeignTable( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('server', None), inobj.pop('options', {})) + obj.fix_privileges() + return obj + + @property + def objtype(self): + return "FOREIGN TABLE" + + def to_map(self, db, opts): + """Convert a foreign table to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return {} + if len(self.columns) == 0: + return {} + cols = [] + for i in range(len(self.columns)): + col = self.columns[i].to_map(db, opts.no_privs) + if col: + cols.append(col) + tbl = {'columns': cols, 'server': self.server} + if self.description is not None: + tbl.update(description=self.description) + if not opts.no_owner and self.owner is not None: + tbl.update(owner=self.owner) + if len(self.options) > 0: + tbl.update(options=self.options) + if not opts.no_privs: + tbl.update({'privileges': self.map_privs()}) + + return tbl + + @grantable + def create(self, dbversion=None): + """Return SQL statements to CREATE the foreign table + + :return: SQL statements + """ + stmts = [] + cols = [] + options = [] + for col in self.columns: + cols.append(" " + col.add()[0]) + if len(self.options) > 0: + options.append(self.options_clause()) + stmts.append("CREATE FOREIGN TABLE %s (\n%s)\n SERVER %s%s" % ( + self.qualname(), ",\n".join(cols), self.server, + options and '\n ' + ',\n '.join(options) or '')) + if self.owner is not None: + stmts.append(self.alter_owner()) + if self.description is not None: + stmts.append(self.comment()) + for col in self.columns: + if col.description is not None: + stmts.append(col.comment()) + return stmts + + def drop(self): + """Return a SQL DROP statement for the foreign table + + :return: SQL statement + """ + return ["DROP %s %s" % (self.objtype, self.identifier())] + + def alter(self, intable): + """Generate SQL to transform an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + """ + stmts = super(ForeignTable, self).alter(intable) + if intable.owner is not None: + if intable.owner != self.owner: + stmts.append(self.alter_owner(intable.owner)) + stmts.append(self.diff_description(intable)) + return stmts + + +class ForeignTableDict(ClassDict): + "The collection of foreign tables in a database" + + cls = ForeignTable + + def _from_catalog(self): + """Initialize the dictionary of tables by querying the catalogs""" + for tbl in self.fetch(): + self[tbl.key()] = tbl + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + for key in inobjs: + if not key.startswith('foreign table '): + raise KeyError("Unrecognized object type: %s" % key) + ftb = key[14:] + inobj = inobjs[key] + self[(schema.name, ftb)] = ftable = ForeignTable.from_map( + ftb, schema, inobj) + try: + newdb.columns.from_map(ftable, inobj['columns']) + except KeyError as exc: + exc.args = ("Foreign table '%s' has no columns" % ftb, ) + raise + + def link_refs(self, dbcolumns): + """Connect columns to their respective foreign tables + + :param dbcolumns: dictionary of columns + """ + for (sch, tbl) in dbcolumns: + if (sch, tbl) in self: + assert isinstance(self[(sch, tbl)], ForeignTable) + self[(sch, tbl)].columns = dbcolumns[(sch, tbl)] + for col in dbcolumns[(sch, tbl)]: + col._table = self[(sch, tbl)] diff --git a/pyrseas/dbobject/function.py b/pyrseas/dbobject/function.py new file mode 100644 index 0000000..2356261 --- /dev/null +++ b/pyrseas/dbobject/function.py @@ -0,0 +1,732 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.function + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines four classes: Proc derived from + DbSchemaObject, Function and Aggregate derived from Proc, and + FunctionDict derived from DbObjectDict. +""" +from pyrseas.yamlutil import MultiLineStr +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, grantable, split_schema_obj + +VOLATILITY_TYPES = {'i': 'immutable', 's': 'stable', 'v': 'volatile'} +PARALLEL_SAFETY = {'r': 'restricted', 's': 'safe', 'u': 'unsafe'} + + +def split_schema_func(schema, func): + """Split a function related to an object from its schema + + :param schema: schema to which the main object belongs + :param func: possibly qualified function name + :returns: a schema, function tuple, or just the unqualified function name + """ + (sch, fnc) = split_schema_obj(func, schema) + if sch != schema: + return (sch, fnc) + else: + return fnc + + +def join_schema_func(func): + """Join the schema and function, if needed, to form a qualified name + + :param func: a schema, function tuple, or just an unqualified function name + :returns: a possibly-qualified schema.function string + """ + if isinstance(func, tuple): + return "%s.%s" % func + else: + return func + + +class Proc(DbSchemaObject): + """A procedure such as a FUNCTION or an AGGREGATE""" + + keylist = ['schema', 'name', 'arguments'] + catalog = 'pg_proc' + + @property + def allprivs(self): + return 'X' + + def __init__(self, name, schema, description, owner, privileges, + arguments): + """Initialize the procedure + + :param name: function name (from proname) + :param schema: schema name (from pronamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via proowner) + :param privileges: access privileges (from proacl) + :param arguments: argument list (without default values, from + pg_function_identity_arguments) + """ + super(Proc, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + self.arguments = arguments + + def extern_key(self): + """Return the key to be used in external maps for this function + + :return: string + """ + return '%s %s(%s)' % (self.objtype.lower(), self.name, self.arguments) + + def identifier(self): + """Return a full identifier for a function object + + :return: string + """ + return "%s(%s)" % (self.qualname(), self.arguments) + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Proc, self).get_implied_deps(db) + + # Add back the language + if isinstance(self, Function) and getattr(self, 'language', None): + lang = db.languages.get(self.language) + if lang: + deps.add(lang) + + # Add back the types + if self.arguments: + for arg in self.arguments.split(', '): + arg = db.find_type(arg.split()[-1]) + if arg is not None: + deps.add(arg) + + return deps + + +class Function(Proc): + """A procedural language function""" + + def __init__(self, name, schema, description, owner, privileges, + arguments, language, returns, source, obj_file=None, + configuration=None, volatility=None, leakproof=False, + strict=False, security_definer=False, cost=0, rows=0, + allargs=None, oid=None): + """Initialize the function + + :param name-arguments: see Proc.__init__ params + :param language: implementation language (from prolang) + :param returns: return type (from pg_get_function_result/prorettype) + :param source: source code, link symbol, etc. (from prosrc) + :param obj_file: language-specific info (from probin) + :param configuration: configuration variables (from proconfig) + :param volatility: volatility type (from provolatile) + :param leakproof: has side effects (from proleakproof) + :param strict: null handling (from proisstrict) + :param security_definer: security definer (from prosecdef) + :param cost: execution cost estimate (from procost) + :param rows: result row estimate (from prorows) + :param allargs: argument list with defaults (from + pg_get_function_arguments) + """ + super(Function, self).__init__( + name, schema, description, owner, privileges, arguments) + self.language = language + self.returns = returns + if source and '\n' in source: + newsrc = [] + for line in source.split('\n'): + if line and line[-1] in (' ', '\t'): + line = line.rstrip() + newsrc.append(line) + source = '\n'.join(newsrc) + self.source = MultiLineStr(source) + self.obj_file = obj_file + self.configuration = configuration + self.allargs = allargs + if volatility is not None: + self.volatility = volatility[:1].lower() + else: + self.volatility = 'v' + assert self.volatility in VOLATILITY_TYPES.keys() + self.leakproof = leakproof + self.strict = strict + self.security_definer = security_definer + self.cost = cost + self.rows = rows + self.oid = oid + + @staticmethod + def query(dbversion=None): + query = """ + SELECT nspname AS schema, proname AS name, + pg_get_function_identity_arguments(p.oid) AS arguments, + pg_get_function_arguments(p.oid) AS allargs, + pg_get_function_result(p.oid) AS returns, rolname AS owner, + array_to_string(proacl, ',') AS privileges, + l.lanname AS language, provolatile AS volatility, + proisstrict AS strict, prosrc AS source, + probin::text AS obj_file, proconfig AS configuration, + prosecdef AS security_definer, procost AS cost, + proleakproof AS leakproof, prorows::integer AS rows, + obj_description(p.oid, 'pg_proc') AS description, p.oid + FROM pg_proc p JOIN pg_roles r ON (r.oid = proowner) + JOIN pg_namespace n ON (pronamespace = n.oid) + JOIN pg_language l ON (prolang = l.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND %s + AND p.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY nspname, proname""" + if dbversion < 110000: + query = query % "NOT proisagg" + else: + query = query % "prokind = 'f'" + return query + + @staticmethod + def from_map(name, schema, arguments, inobj): + """Initialize a function instance from a YAML map + + :param name: function name + :param name: schema name + :param arguments: arguments + :param inobj: YAML map of the function + :return: function instance + """ + src = inobj.get('source', None) + objfile = inobj.get('obj_file', None) + if (src and objfile) or not (src or objfile): + raise ValueError("Function '%s': either source or obj_file must " + "be specified" % name) + obj = Function( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + arguments, inobj.pop('language', None), + inobj.pop('returns', None), inobj.pop('source', None), + inobj.pop('obj_file', None), + inobj.pop('configuration', None), + inobj.pop('volatility', None), + inobj.pop('leakproof', False), inobj.pop('strict', False), + inobj.pop('security_definer', False), + inobj.pop('cost', 0), inobj.pop('rows', 0), + inobj.pop('allargs', None)) + obj.fix_privileges() + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert a function to a YAML-suitable format + + :param no_owner: exclude function owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(Function, self).to_map(db, no_owner, no_privs) + for attr in ('leakproof', 'strict', 'security_definer'): + if dct[attr] is False: + dct.pop(attr) + if self.allargs is None or len(self.allargs) == 0 or \ + self.allargs == self.arguments: + dct.pop('allargs') + if self.configuration is None: + dct.pop('configuration') + if self.volatility == 'v': + dct.pop('volatility') + else: + dct['volatility'] = VOLATILITY_TYPES[self.volatility] + if self.obj_file is not None: + dct['link_symbol'] = self.source + del dct['source'] + else: + del dct['obj_file'] + if self.cost != 0: + if self.language in ['c', 'internal']: + if self.cost == 1: + del dct['cost'] + else: + if self.cost == 100: + del dct['cost'] + else: + del dct['cost'] + if self.rows != 0: + if self.rows == 1000: + del dct['rows'] + else: + del dct['rows'] + + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newsrc=None, basetype=False, returns=None): + """Return SQL statements to CREATE or REPLACE the function + + :param newsrc: new source for a changed function + :return: SQL statements + """ + stmts = [] + if self.obj_file is not None: + src = "'%s', '%s'" % (self.obj_file, + hasattr(self, 'link_symbol') and + self.link_symbol or self.name) + elif self.language == 'internal': + src = "$$%s$$" % (newsrc or self.source) + else: + src = "$_$%s$_$" % (newsrc or self.source) + volat = leakproof = strict = secdef = cost = rows = config = '' + if self.volatility != 'v': + volat = ' ' + VOLATILITY_TYPES[self.volatility].upper() + if self.leakproof is True: + leakproof = ' LEAKPROOF' + if self.strict: + strict = ' STRICT' + if self.security_definer: + secdef = ' SECURITY DEFINER' + if self.configuration is not None: + config = ' SET %s' % self.configuration[0] + if self.cost != 0: + if self.language in ['c', 'internal']: + if self.cost != 1: + cost = " COST %s" % self.cost + else: + if self.cost != 100: + cost = " COST %s" % self.cost + if self.rows != 0: + if self.rows != 1000: + rows = " ROWS %s" % self.rows + + # We may have to create a shell type if we are its input or output + # functions + t = getattr(self, '_defining', None) + if t is not None: + if not hasattr(t, '_shell_created'): + t._shell_created = True + stmts.append("CREATE TYPE %s" % t.qualname()) + + if self.allargs is not None: + args = self.allargs + elif self.arguments is not None: + args = self.arguments + else: + args = '' + stmts.append("CREATE%s FUNCTION %s(%s) RETURNS %s\n LANGUAGE %s" + "%s%s%s%s%s%s%s\n AS %s" % ( + newsrc and " OR REPLACE" or '', self.qualname(), + args, returns or self.returns, self.language, volat, leakproof, + strict, secdef, cost, rows, config, src)) + return stmts + + def alter(self, infunction, dbversion=None, no_owner=False): + """Generate SQL to transform an existing function + + :param infunction: a YAML map defining the new function + :return: list of SQL statements + + Compares the function to an input function and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if self.source != infunction.source and infunction.source is not None: + stmts.append(self.create( + dbversion=dbversion, + returns=infunction.returns, + newsrc=infunction.source, + )) + if self.leakproof is True: + if infunction.leakproof is True: + stmts.append("ALTER FUNCTION %s LEAKPROOF" % self.identifier()) + else: + stmts.append("ALTER FUNCTION %s NOT LEAKPROOF" + % self.identifier()) + elif infunction.leakproof is True: + stmts.append("ALTER FUNCTION %s LEAKPROOF" % self.qualname()) + stmts.append(super(Function, self).alter(infunction, + no_owner=no_owner)) + return stmts + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Function, self).get_implied_deps(db) + + # Add back the return type + rettype = self.returns + if rettype.upper().startswith("SETOF "): + rettype = rettype.split(None, 1)[-1] + rettype = db.find_type(rettype) + if rettype is not None: + deps.add(rettype) + + return deps + + def get_deps(self, db): + deps = super(Function, self).get_deps(db) + + # avoid circular import dependencies + from .dbtype import DbType + + # drop the dependency on the type if this function is an in/out + # because there is a loop here. + for dep in list(deps): + if isinstance(dep, DbType): + for attr in ('input', 'output', 'send', 'receive'): + fname = getattr(dep, attr, None) + if isinstance(fname, tuple): + fname = "%s.%s" % fname + else: + fname = "%s.%s" % (self.schema, fname) + if fname and fname == self.qualname(): + deps.remove(dep) + self._defining = dep # we may need a shell for this + break + + return deps + + def drop(self): + """Generate SQL to drop the current function + + :return: list of SQL statements + """ + # If the function defines a type it will be dropped by the CASCADE + # on the type. + if getattr(self, '_defining', None): + return [] + else: + return super(Function, self).drop() + + +AGGREGATE_KINDS = {'n': 'normal', 'o': 'ordered', 'h': 'hypothetical'} + + +class Aggregate(Proc): + """An aggregate function""" + + def __init__(self, name, schema, description, owner, privileges, + arguments, sfunc, stype, sspace=0, finalfunc=None, + finalfunc_extra=False, initcond=None, sortop=None, + msfunc=None, minvfunc=None, mstype=None, msspace=0, + mfinalfunc=None, mfinalfunc_extra=False, minitcond=None, + kind='normal', combinefunc=None, serialfunc=None, + deserialfunc=None, parallel='unsafe', + oid=None): + """Initialize the aggregate + + :param name-arguments: see Proc.__init__ params + :param sfunc: state transition function (from aggtransfn) + :param stype: state datatype (from aggtranstype) + :param sspace: transition state data size (from aggtransspace) + :param finalfunc: final function (from aggfinalfn) + :param finalfunc_extra: extra args? (from aggfinalextra) + :param initcond: initial value (from agginitval) + :param sortop: sort operator (from aggsortop) + :param msfunc: state transition function (from aggmtransfn) + :param minvfunc: inverse transition function (from aggminvtransfn) + :param mstype: state datatype (from aggmtranstype) + :param msspace: transition state data size (from aggmtransspace) + :param mfinalfunc: final function (from aggfinalfn) + :param mfinalfunc_extra: extra args? (from aggmfinalextra) + :param minitcond: initial value (from aggminitval) + :param kind: aggregate kind (from aggkind) + :param combinefunc: combine function (from aggcombinefn) + :param serialfunc: serialization function (from aggserialfn) + :param deserialfunc: deserialization function (from aggdeserialfn) + :param parallel: parallel safety indicator (from proparallel) + """ + super(Aggregate, self).__init__( + name, schema, description, owner, privileges, arguments) + self.sfunc = split_schema_obj(sfunc, self.schema) + self.stype = self.unqualify(stype) + self.sspace = sspace + if finalfunc is not None and finalfunc != '-': + self.finalfunc = split_schema_obj(finalfunc, self.schema) + else: + self.finalfunc = None + self.finalfunc_extra = finalfunc_extra + self.initcond = initcond + self.sortop = sortop if sortop != '0' else None + if msfunc is not None and msfunc != '-': + self.msfunc = split_schema_obj(msfunc, self.schema) + else: + self.msfunc = None + if minvfunc is not None and minvfunc != '-': + self.minvfunc = split_schema_obj(minvfunc, self.schema) + else: + self.minvfunc = None + if mstype is not None and mstype != '-': + self.mstype = self.unqualify(mstype) + else: + self.mstype = None + self.msspace = msspace + if mfinalfunc is not None and mfinalfunc != '-': + self.mfinalfunc = split_schema_obj(mfinalfunc, self.schema) + else: + self.mfinalfunc = None + self.mfinalfunc_extra = mfinalfunc_extra + self.minitcond = minitcond + if kind is None: + self.kind = 'normal' + elif len(kind) == 1: + self.kind = AGGREGATE_KINDS[kind] + else: + self.kind = kind + assert self.kind in AGGREGATE_KINDS.values() + self.combinefunc = combinefunc if combinefunc != '-' else None + self.serialfunc = serialfunc if serialfunc != '-' else None + self.deserialfunc = deserialfunc if deserialfunc != '-' else None + if parallel is None: + self.parallel = 'unsafe' + elif len(parallel) == 1: + self.parallel = PARALLEL_SAFETY[parallel] + else: + self.parallel = parallel + assert self.parallel in PARALLEL_SAFETY.values() + self.oid = oid + + @staticmethod + def query(dbversion): + query = """ + SELECT nspname AS schema, proname AS name, + pg_get_function_identity_arguments(p.oid) AS arguments, + rolname AS owner, + array_to_string(proacl, ',') AS privileges, + aggtransfn::regproc AS sfunc, + aggtranstype::regtype AS stype, aggtransspace AS sspace, + aggfinalfn::regproc AS finalfunc, + aggfinalextra AS finalfunc_extra, + agginitval AS initcond, aggsortop::regoper AS sortop, + aggmtransfn::regproc AS msfunc, + aggminvtransfn::regproc AS minvfunc, + aggmtranstype::regtype AS mstype, + aggmtransspace AS msspace, + aggmfinalfn::regproc AS mfinalfunc, + aggmfinalextra AS mfinalfunc_extra, + aggminitval AS minitcond, aggkind AS kind, + aggcombinefn AS combinefunc, + aggserialfn AS serialfunc, aggdeserialfn AS deserialfunc, + proparallel AS parallel,%s + obj_description(p.oid, 'pg_proc') AS description, p.oid + FROM pg_proc p JOIN pg_roles r ON (r.oid = proowner) + JOIN pg_namespace n ON (pronamespace = n.oid) + LEFT JOIN pg_aggregate a ON (p.oid = aggfnoid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + %s + AND p.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY nspname, proname""" + extra = ("", "") + if dbversion < 110000: + extra = (" proisagg,", "") + else: + extra = ("", "AND prokind = 'a'") + return query % extra + + @staticmethod + def from_map(name, schema, arguments, inobj): + """Initialize an aggregate instance from a YAML map + + :param name: aggregate name + :param name: schema name + :param arguments: arguments + :param inobj: YAML map of the aggregate + :return: aggregate instance + """ + obj = Aggregate( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + arguments, inobj.get('sfunc'), inobj.get('stype'), + inobj.pop('sspace', 0), inobj.pop('finalfunc', None), + inobj.pop('finalfunc_extra', False), inobj.pop('initcond', None), + inobj.pop('sortop', None), inobj.pop('msfunc', None), + inobj.pop('minvfunc', None), inobj.pop('mstype', None), + inobj.pop('msspace', 0), inobj.pop('mfinalfunc', None), + inobj.pop('mfinalfunc_extra', False), + inobj.pop('minitcond', None), inobj.pop('kind', 'normal'), + inobj.pop('combinefunc', None), inobj.pop('serialfunc', None), + inobj.pop('deseriafunc', None), inobj.pop('parallel', 'unsafe')) + obj.fix_privileges() + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert an aggregate to a YAML-suitable format + + :param no_owner: exclude aggregate owner information + :param no_privs: exclude privilege information + :return: dictionary + """ + dct = super(Aggregate, self).to_map(db, no_owner, no_privs) + dct['sfunc'] = self.unqualify(join_schema_func(self.sfunc)) + for attr in ('finalfunc', 'msfunc', 'minvfunc', 'mfinalfunc'): + if getattr(self, attr) is None: + dct.pop(attr) + else: + dct[attr] = self.unqualify( + join_schema_func(getattr(self, attr))) + for attr in ('initcond', 'sortop', 'minitcond', 'mstype', + 'combinefunc', 'serialfunc', 'deserialfunc'): + if getattr(self, attr) is None: + dct.pop(attr) + for attr in ('sspace', 'msspace'): + if getattr(self, attr) == 0: + dct.pop(attr) + for attr in ('finalfunc_extra', 'mfinalfunc_extra'): + if getattr(self, attr) is False: + dct.pop(attr) + if self.kind == 'normal': + dct.pop('kind') + if self.parallel == 'unsafe': + dct.pop('parallel') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the aggregate + + :param dbversion: Posgres version + :return: SQL statements + """ + opt_clauses = [] + if self.finalfunc is not None: + opt_clauses.append("FINALFUNC = %s" % + join_schema_func(self.finalfunc)) + if self.initcond is not None: + opt_clauses.append("INITCOND = '%s'" % self.initcond) + if self.combinefunc is not None: + opt_clauses.append("COMBINEFUNC = %s" % self.combinefunc) + if self.serialfunc is not None: + opt_clauses.append("SERIALFUNC = %s" % self.serialfunc) + if self.deserialfunc is not None: + opt_clauses.append("DESERIALFUNC = %s" % self.deserialfunc) + if self.sspace > 0: + opt_clauses.append("SSPACE = %d" % self.sspace) + if self.finalfunc_extra: + opt_clauses.append("FINALFUNC_EXTRA") + if self.msfunc is not None: + opt_clauses.append("MSFUNC = %s" % join_schema_func(self.msfunc)) + if self.minvfunc is not None: + opt_clauses.append("MINVFUNC = %s" % join_schema_func(self.minvfunc)) + if self.mstype is not None: + opt_clauses.append("MSTYPE = %s" % self.mstype) + if self.msspace > 0: + opt_clauses.append("MSSPACE = %d" % self.msspace) + if self.mfinalfunc is not None: + opt_clauses.append("MFINALFUNC = %s" % + join_schema_func(self.mfinalfunc)) + if self.mfinalfunc_extra: + opt_clauses.append("MFINALFUNC_EXTRA") + if self.minitcond is not None: + opt_clauses.append("MINITCOND = '%s'" % self.minitcond) + if self.kind == 'hypothetical': + opt_clauses.append("HYPOTHETICAL") + if self.sortop is not None: + clause = self.sortop + if not clause.startswith('OPERATOR'): + clause = "OPERATOR(%s)" % clause + opt_clauses.append("SORTOP = %s" % clause) + if self.parallel != 'unsafe': + opt_clauses.append("PARALLEL = %s" % self.parallel.upper()) + return ["CREATE AGGREGATE %s(%s) (\n SFUNC = %s," + "\n STYPE = %s%s%s)" % ( + self.qualname(), self.arguments, + join_schema_func(self.sfunc), self.stype, + opt_clauses and ',\n ' or '', + ',\n '.join(opt_clauses))] + + def get_implied_deps(self, db): + # List the previous dependencies + deps = super(Aggregate, self).get_implied_deps(db) + + if isinstance(self.sfunc, tuple): + sch, fnc = self.sfunc + else: + sch, fnc = self.schema, self.sfunc + if 'ORDER BY' in self.arguments: + args = self.arguments.replace(' ORDER BY', ',') + else: + args = self.stype + ', ' + self.arguments + deps.add(db.functions[sch, fnc, args]) + for fn in ('finalfunc', 'mfinalfunc'): + if getattr(self, fn) is not None: + func = getattr(self, fn) + if isinstance(func, tuple): + sch, fnc = func + else: + sch, fnc = self.schema, func + deps.add(db.functions[sch, fnc, self.mstype + if fn[0] == 'm' else self.stype]) + for fn in ('msfunc', 'minvfunc'): + if getattr(self, fn) is not None: + func = getattr(self, fn) + if isinstance(func, tuple): + sch, fnc = func + else: + sch, fnc = self.schema, func + args = self.mstype + ", " + self.arguments + deps.add(db.functions[sch, fnc, args]) + + return deps + + +class ProcDict(DbObjectDict): + "The collection of regular and aggregate functions in a database" + + cls = Proc + + def _from_catalog(self): + """Initialize the dictionary of procedures by querying the catalogs""" + for cls in (Function, Aggregate): + self.cls = cls + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, infuncs): + """Initialize the dictionary of functions by converting the input map + + :param schema: schema owning the functions + :param infuncs: YAML map defining the functions + """ + for key in infuncs: + (objtype, spc, fnc) = key.partition(' ') + if spc != ' ' or objtype not in ['function', 'aggregate']: + raise KeyError("Unrecognized object type: %s" % key) + paren = fnc.find('(') + if paren == -1 or fnc[-1:] != ')': + raise KeyError("Invalid function signature: %s" % fnc) + arguments = fnc[paren + 1:-1] + inobj = infuncs[key] + fnc = fnc[:paren] + if objtype == 'function': + func = Function.from_map(fnc, schema, arguments, inobj) + else: + func = Aggregate.from_map(fnc, schema, arguments, inobj) + self[(schema.name, fnc, arguments)] = func + + def find(self, func, args): + """Return a function given its name and arguments + + :param func: name of the function, eventually with schema + :param args: list of type names + + Return the function found, else None. + """ + schema, name = split_schema_obj(func) + args = ', '.join(args) + return self.get((schema, name, args)) + + def link_refs(self, dbtypes): + """Connect the functions to other objects + + - Connect defining functions to the type they define + + :param dbtypes: dictionary of types + """ + # TODO: this link is needed from map, not from sql. + # is this a pattern? I was assuming link_refs would have disappeared + # but I'm actually still maintaining them. Verify if they are always + # only used for from_map, not for from_catalog + for key in dbtypes: + t = dbtypes[key] + for f in t.find_defining_funcs(self): + f._defining = t diff --git a/pyrseas/dbobject/index.py b/pyrseas/dbobject/index.py new file mode 100644 index 0000000..c87c938 --- /dev/null +++ b/pyrseas/dbobject/index.py @@ -0,0 +1,353 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.index + ~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Index and IndexDict, derived + from DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable + + +def split_exprs(idx_exprs): + "Helper function to split index expressions from pg_get_expr()" + level = 0 + in_literal = False + splits = [] + + # Split around commas but only at the first parens level. + # TODO: you can still fool this function with a string containing a quote. + start = 0 + for i, c in enumerate(idx_exprs): + if c == "'": + in_literal = not in_literal + continue + if in_literal: + continue + elif c == '(': + level += 1 + elif c == ')': + level -= 1 + elif c == ',' and level == 0: + splits.append((start, i)) + start = i + 1 + for s in idx_exprs[start:]: + if s == ' ': + start += 1 + else: + break + splits.append((start, i+1)) + return [idx_exprs[start:end] for start, end in splits] + + +class Index(DbSchemaObject): + """A physical index definition, other than a primary key or unique + constraint index. + + An index is identified by its schema name and index name. However, + at this time, Pyrseas uses the triple schema-table-index names as the + identifier. + """ + # TODO: This should be fixed in this or a subsequent release. + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_index' + + def __init__(self, name, schema, table, description, unique=False, + access_method='btree', keys=[], predicate=None, + tablespace=None, cluster=False, keyexprs=None, defn=None, + oid=None): + """Initialize the index + + :param name: index name (from relname) + :param schema: schema name (from nspname via relnamespace) + :param table: table name (from indrelid) + :param description: comment text (from obj_description) + :param unique: unique indicator (from indisunique) + :param access_method: access method (from amname via relam) + :param keys: list of columns (from indkey) + :param predicate: partial index predicate (from indpred) + :param tablespace: tablespace name (from spcname via reltablespace) + :param cluster: clustered indicator (from indisclustered) + :param keyexprs: list of expressions (from indexprs) + :param defn: index definition (from pg_get_indexdef) + """ + super(Index, self).__init__(name, schema, description) + self.table = self.unqualify(table) + self.unique = unique + self.access_method = access_method + if defn is not None: + self.keys = self._parse_keys(keys, keyexprs, defn) + else: + self.keys = keys + self.predicate = predicate + self.tablespace = tablespace + self.cluster = cluster + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, indrelid::regclass AS table, + c.relname AS name, amname AS access_method, + indisunique AS unique, indkey AS keys, + pg_get_expr(indexprs, indrelid) AS keyexprs, + pg_get_expr(indpred, indrelid) AS predicate, + pg_get_indexdef(indexrelid) AS defn, + spcname AS tablespace, indisclustered AS cluster, + obj_description (c.oid, 'pg_class') AS description, c.oid + FROM pg_index i JOIN pg_class c ON (indexrelid = c.oid) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + JOIN pg_am ON (relam = pg_am.oid) + LEFT JOIN pg_tablespace t ON (c.reltablespace = t.oid) + WHERE NOT indisprimary AND c.relpersistence != 't' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE contype in ('p', 'u') + AND conindid = c.oid) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize an index instance from a YAML map + + :param name: index name + :param table: map of table + :param inobj: YAML map of the index + :return: Index instance + """ + keys = 'keys' + if 'columns' in inobj: + keys = 'columns' + elif 'keys' not in inobj: + raise KeyError("Index '%s' is missing keys specification" % name) + obj = Index( + name, table.schema, table.name, inobj.pop('description', None), + inobj.pop('unique', False), inobj.pop('access_method', 'btree'), + inobj.pop(keys, []), inobj.pop('predicate', None), + inobj.pop('tablespace', None), inobj.pop('cluster', False)) + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + obj.set_oldname(inobj) + return obj + + def _parse_keys(self, keycols, exprs, defn): + keydefs, _, _ = defn.partition(' WHERE ') + _, _, keydefs = keydefs.partition(' USING ') + keydefs = keydefs[keydefs.find(' (') + 2:-1] + # split expressions + if exprs is not None: + exprs = split_exprs(exprs) + i = 0 + rest = keydefs + keys = [] + for col in keycols.split(): + keyopts = [] + extra = {} + if col == '0': + expr = exprs[i] + if rest and rest[0] == '(': + expr = '(' + expr + ')' + assert(rest.startswith(expr)) + key = expr + extra = {'type': 'expression'} + explen = len(expr) + loc = rest[explen:].find(',') + if loc == 0: + keyopts = [] + rest = rest[explen + 1:].lstrip() + elif loc == -1: + keyopts = rest[explen:].split() + rest = '' + else: + keyopts = rest[explen:explen + loc].split() + rest = rest[explen + loc + 1:].lstrip() + i += 1 + else: + loc = rest.find(',') + key = rest[:loc] if loc != -1 else rest.lstrip() + keyopts = key.split()[1:] + key = key.split()[0] + rest = rest[loc + 1:] + rest = rest.lstrip() + skipnext = False + for j, opt in enumerate(keyopts): + if skipnext: + skipnext = False + continue + if opt.upper() not in ['COLLATE', 'ASC', 'DESC', 'NULLS', + 'FIRST', 'LAST']: + extra.update(opclass=opt) + continue + elif opt == 'COLLATE': + extra.update(collation=keyopts[j + 1]) + skipnext = True + elif opt == 'NULLS': + extra.update(nulls=keyopts[j + 1].lower()) + skipnext = True + elif opt == 'DESC': + extra.update(order='desc') + if extra: + key = {key: extra} + keys.append(key) + return keys + + def key_expressions(self): + """Return comma-separated list of key column names and qualifiers + + :return: string + """ + colspec = [] + for col in self.keys: + if isinstance(col, str): + colspec.append(col) + else: + clause = list(col.keys())[0] + vals = list(col.values())[0] + if 'collation' in vals: + clause += ' COLLATE ' + vals['collation'] + if 'opclass' in vals: + clause += ' ' + vals['opclass'] + if 'order' in vals: + clause += ' ' + vals['order'].upper() + if 'nulls' in vals: + clause += ' NULLS ' + vals['nulls'].upper() + colspec.append(clause) + return ", ".join(colspec) + + def to_map(self, db): + """Convert an index definition to a YAML-suitable format + + :return: dictionary + """ + dct = super(Index, self).to_map(db) + if self.access_method == 'btree': + dct.pop('access_method') + if not self.unique: + dct.pop('unique') + for attr in ['predicate', 'tablespace']: + if getattr(self, attr) is None: + dct.pop(attr) + if not self.cluster: + dct.pop('cluster') + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return a SQL statement to CREATE the index + + :return: SQL statements + """ + stmts = [] + + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return stmts + + acc = '' + if self.access_method != 'btree': + acc = 'USING %s ' % self.access_method + tblspc = '' + if self.tablespace is not None: + tblspc = '\n TABLESPACE %s' % self.tablespace + pred = '' + if self.predicate is not None: + pred = '\n WHERE %s' % self.predicate + stmts.append("CREATE %sINDEX %s ON %s %s(%s)%s%s" % ( + 'UNIQUE ' if self.unique else '', quote_id(self.name), + self.qualname(self.schema, self.table), acc, + self.key_expressions(), tblspc, pred)) + if self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self.qualname(self.schema, self.table), quote_id(self.name))) + return stmts + + def alter(self, inindex): + """Generate SQL to transform an existing index + + :param inindex: a YAML map defining the new index + :return: list of SQL statements + + Compares the index to an input index and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return stmts + + if self.access_method != inindex.access_method \ + or self.unique != inindex.unique \ + or self.keys != inindex.keys: + stmts.append("DROP INDEX %s" % self.qualname()) + self.access_method = inindex.access_method + self.unique = inindex.unique + self.keys = inindex.keys + stmts.append(self.create()) + + base = "ALTER INDEX %s\n " % self.qualname() + if inindex.tablespace is not None: + if self.tablespace is not None \ + or self.tablespace != inindex.tablespace: + stmts.append(base + "SET TABLESPACE %s" + % quote_id(inindex.tablespace)) + elif self.tablespace is not None: + stmts.append(base + "SET TABLESPACE pg_default") + if inindex.cluster: + if not self.cluster: + stmts.append("CLUSTER %s USING %s" % ( + self.qualname(self.schema, self.table), + quote_id(self.name))) + elif self.cluster: + stmts.append("ALTER TABLE %s\n SET WITHOUT CLUSTER" % + self.qualname(self.schema, self.table)) + stmts.append(super(Index, self).alter(inindex)) + return stmts + + def drop(self): + """Generate SQL to drop the current index + + :return: list of SQL statements + """ + # indexes defined by constraints are not to be dealt with as indexes + if getattr(self, '_for_constraint', None): + return [] + + return ["DROP INDEX %s" % self.identifier()] + + def get_implied_deps(self, db): + deps = super(Index, self).get_implied_deps(db) + + # add the table we are defined into + deps.add(db.tables[self.schema, self.table]) + # TODO: add column collation specs if present + + return deps + + +class IndexDict(DbObjectDict): + "The collection of indexes on tables in a database" + + cls = Index + + def _from_catalog(self): + """Initialize the dictionary of indexes by querying the catalogs""" + self.query = self.cls.query() + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, table, inindexes): + """Initialize the dictionary of indexes by converting the input map + + :param table: table owning the indexes + :param inindexes: YAML map defining the indexes + """ + for i in inindexes: + inobj = inindexes[i] + self[(table.schema, table.name, i)] = Index.from_map( + i, table, inobj) diff --git a/pyrseas/dbobject/language.py b/pyrseas/dbobject/language.py new file mode 100644 index 0000000..9f17b10 --- /dev/null +++ b/pyrseas/dbobject/language.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.language + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Language and LanguageDict, derived from + DbObject and DbObjectDict, respectively. + + See note at + https://www.postgresql.org/docs/current/static/sql-createlanguage.html + regarding status of procedural languages since Postgres 9.1. +""" +from . import DbObjectDict, DbObject, quote_id +from .function import Function +from .extension import CORE_LANGS + + +class Language(DbObject): + """A procedural language definition""" + + keylist = ['name'] + single_extern_file = True + catalog = 'pg_language' + + def __init__(self, name, description=None, owner=None, privileges=[], + trusted=False, + oid=None): + """Initialize the language + + :param name: language name (from lanname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via lanowner) + :param privileges: access privileges (from lanacl) + :param trusted: is this a trusted language? (from lanpltrusted) + """ + super(Language, self).__init__(name, description) + self._init_own_privs(owner, privileges) + self.trusted = trusted + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT lanname AS name, lanpltrusted AS trusted, rolname AS owner, + array_to_string(lanacl, ',') AS privileges, + obj_description(l.oid, 'pg_language') AS description, l.oid + FROM pg_language l JOIN pg_roles r ON (r.oid = lanowner) + WHERE lanispl + ORDER BY lanname""" + + @staticmethod + def from_map(name, inobj): + """Initialize a Language instance from a YAML map + + :param name: Language name + :param inobj: YAML map of the Language + :return: Language instance + """ + obj = Language( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('privileges', []), inobj.pop('trusted', False)) + obj.fix_privileges() + if '_ext' in inobj: + obj._ext = inobj['_ext'] + obj.set_oldname(inobj) + return obj + + def to_map(self, db, no_owner, no_privs): + """Convert language to a YAML-suitable format + + :param no_owner: exclude language owner information + :return: dictionary + """ + if hasattr(self, '_ext'): + return None + if self.name in CORE_LANGS: + return None + dct = super(Language, self).to_map(db, no_owner, no_privs) + if 'functions' in dct: + del dct['functions'] + return dct + + def create(self, dbversion=None): + """Return SQL statements to CREATE the language + + :return: empty, because PL's should all be extensions + """ + return [] + + def alter(self, inobj): + if self.name in CORE_LANGS: + return [] + return super(Language, self).alter(inobj) + + def drop(self): + if self.name in CORE_LANGS: + return [] + return super(Language, self).drop() + + +class LanguageDict(DbObjectDict): + "The collection of procedural languages in a database." + + cls = Language + + def from_map(self, inmap): + """Initialize the dictionary of languages by examining the input map + + :param inmap: the input YAML map defining the languages + """ + for key in inmap: + (objtype, spc, lng) = key.partition(' ') + if spc != ' ' or objtype != 'language': + raise KeyError("Unrecognized object type: %s" % key) + inobj = inmap[key] + self[lng] = Language.from_map(lng, inobj) + + def link_refs(self, dbfunctions, langs): + """Connect functions to their respective languages + + :param dbfunctions: dictionary of functions + + Fills in the `functions` dictionary for each language by + traversing the `dbfunctions` dictionary, which is keyed by + schema and function name. + """ + for (sch, fnc, arg) in dbfunctions: + func = dbfunctions[(sch, fnc, arg)] + if not isinstance(func, Function) or ( + func.language in ['sql', 'c', 'internal']): + continue + try: + language = self[(func.language)] + except KeyError as exc: + if func.language in langs: + continue + raise exc + if not hasattr(language, 'functions'): + language.functions = {} + language.functions.update({fnc: func}) diff --git a/pyrseas/dbobject/operator.py b/pyrseas/dbobject/operator.py new file mode 100644 index 0000000..b55732d --- /dev/null +++ b/pyrseas/dbobject/operator.py @@ -0,0 +1,236 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operator + ~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Operator derived from + DbSchemaObject and OperatorDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, ownable +from . import split_schema_obj, split_func_args + + +class Operator(DbSchemaObject): + """An operator""" + + keylist = ['schema', 'name', 'leftarg', 'rightarg'] + single_extern_file = True + catalog = 'pg_operator' + + def __init__(self, name, schema, description, owner, procedure, + leftarg=None, rightarg=None, commutator=None, negator=None, + restrict=None, join=None, hashes=False, merges=False, + oid=None): + """Initialize the operator + + :param name: operator name (from oprname) + :param description: comment text (from obj_description()) + :param schema: schema name (from oprnamespace) + :param owner: owner name (from rolname via oprowner) + :param procedure: implementor function (from oprcode) + :param leftarg: left operand type (from oprleft) + :param rightarg: right operand type (from oprright) + :param commutator: commutator, if any (from oprcom) + :param negator: negator, if any (from oprnegate) + :param restrict: restriction selectivity function (from oprrest) + :param join: join selectivity function (from oprjoin) + :param hashes: supports hash joins? (from oprcanhash) + :param merges: support merge joins? (from oprcanmerge) + """ + super(Operator, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.procedure = procedure + self.leftarg = leftarg if leftarg != '-' else None + self.rightarg = rightarg if rightarg != '-' else None + self.commutator = commutator if commutator != '0' else None + self.negator = negator if negator != '0' else None + self.restrict = restrict if restrict != '-' else None + self.join = join if join != '-' else None + self.hashes = hashes + self.merges = merges + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, oprname AS name, rolname AS owner, + oprleft::regtype AS leftarg, oprright::regtype AS rightarg, + oprcode AS procedure, oprcom::regoper AS commutator, + oprnegate::regoper AS negator, oprrest AS restrict, + oprjoin AS join, oprcanhash AS hashes, + oprcanmerge AS merges, + obj_description(o.oid, 'pg_operator') AS description, o.oid + FROM pg_operator o JOIN pg_roles r ON (r.oid = oprowner) + JOIN pg_namespace n ON (oprnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_operator'::regclass) + ORDER BY nspname, oprname""" + + @staticmethod + def from_map(name, schema, leftarg, rightarg, inobj): + """Initialize an operator instance from a YAML map + + :param name: operator name + :param name: schema name + :param leftarg: left-hand argument + :param rightarg: right-hand argument + :param inobj: YAML map of the operator + :return: operator instance + """ + obj = Operator( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('procedure', None), + leftarg, rightarg, inobj.pop('commutator', None), + inobj.pop('negator', None), inobj.pop('restrict', None), + inobj.pop('join', None), inobj.pop('hashes', False), + inobj.pop('merges', None)) + obj.set_oldname(inobj) + return obj + + def extern_key(self): + """Return the key to be used in external maps for this operator + + :return: string + """ + return '%s %s(%s, %s)' % ( + self.objtype.lower(), self.name, + 'NONE' if self.leftarg is None else self.leftarg, + 'NONE' if self.rightarg is None else self.rightarg) + + def qualname(self): + """Return the schema-qualified name of the operator + + :return: string + + No qualification is used if the schema is 'public'. + """ + return self.schema == 'public' and self.name \ + or "%s.%s" % (quote_id(self.schema), self.name) + + def identifier(self): + """Return a full identifier for an operator object + + :return: string + """ + return "%s(%s, %s)" % (self.qualname(), self.leftarg, self.rightarg) + + def to_map(self, db, no_owner=False): + """Convert an operator to a YAML-suitable format + + :param db: db used to tie the objects together + :param no_owner: exclude object owner information + :return: dictionary + """ + dct = super(Operator, self).to_map(db, no_owner) + for attr in ['commutator', 'join', 'negator', 'restrict']: + if dct[attr] is None: + dct.pop(attr) + for attr in ['hashes', 'merges']: + if dct[attr] is False: + dct.pop(attr) + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE or REPLACE the operator + + :return: SQL statements + """ + opt_clauses = [] + if self.leftarg is not None: + opt_clauses.append("LEFTARG = %s" % self.leftarg) + if self.rightarg is not None: + opt_clauses.append("RIGHTARG = %s" % self.rightarg) + if self.commutator is not None: + opt_clauses.append("COMMUTATOR = OPERATOR(%s)" % self.commutator) + if self.negator is not None: + opt_clauses.append("NEGATOR = OPERATOR(%s)" % self.negator) + if self.restrict is not None: + opt_clauses.append("RESTRICT = %s" % self.restrict) + if self.join is not None: + opt_clauses.append("JOIN = %s" % self.join) + if self.hashes: + opt_clauses.append("HASHES") + if self.merges: + opt_clauses.append("MERGES") + return ["CREATE OPERATOR %s (\n PROCEDURE = %s%s%s)" % ( + self.qualname(), self.procedure, + ',\n ' if opt_clauses else '', ',\n '.join(opt_clauses))] + + def get_implied_deps(self, db): + deps = super(Operator, self).get_implied_deps(db) + + # Types may be not found because builtin, or the operator unary + if self.leftarg is not None: + leftarg = db.types.find(self.leftarg) + if leftarg: + deps.add(leftarg) + + if self.rightarg is not None: + rightarg = db.types.find(self.rightarg) + if rightarg: + deps.add(rightarg) + + # The function instead we expect it exists + # TODO: another ugly hack to locate the object + fschema, fname = split_schema_obj(self.procedure, self.schema) + fargs = ', '.join(t for t in [self.leftarg, self.rightarg] + if t is not None) + if (fschema, fname, fargs) in db.functions: + func = db.functions[fschema, fname, fargs] + deps.add(func) + + # This helper function may be a builtin + if self.restrict is not None: + fschema, fname = split_schema_obj(self.restrict) + func = db.functions.get((fschema, fname, + "internal, oid, internal, integer")) + if func: + deps.add(func) + + return deps + + +class OperatorDict(DbObjectDict): + "The collection of operators in a database" + + cls = Operator + + def find(self, oper): + """Return an operator given its signature + + :param oper: a signature such as '#>=#(hstore,hstore)' + + Return the operator found, else None. + """ + schema, name = split_schema_obj(oper) + name, args = split_func_args(name) + return self.get((schema, name) + tuple(args)) + + def from_map(self, schema, inopers): + """Initialize the dictionary of operators by converting the input map + + :param schema: schema owning the operators + :param inopers: YAML map defining the operators + """ + for key in inopers: + (objtype, spc, opr) = key.partition(' ') + if spc != ' ' or objtype != 'operator': + raise KeyError("Unrecognized object type: %s" % key) + paren = opr.find('(') + if paren == -1 or opr[-1:] != ')': + raise KeyError("Invalid operator signature: %s" % opr) + (leftarg, rightarg) = opr[paren + 1:-1].split(',') + if leftarg == 'NONE': + leftarg = None + rightarg = rightarg.lstrip() + if rightarg == 'NONE': + rightarg = None + inobj = inopers[key] + opr = opr[:paren] + self[(schema.name, opr, leftarg, rightarg)] = Operator.from_map( + opr, schema, leftarg, rightarg, inobj) diff --git a/pyrseas/dbobject/operclass.py b/pyrseas/dbobject/operclass.py new file mode 100644 index 0000000..5d874b3 --- /dev/null +++ b/pyrseas/dbobject/operclass.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operclass + ~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: OperatorClass derived from + DbSchemaObject and OperatorClassDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_func_args, split_schema_obj + + +class OperatorClass(DbSchemaObject): + """An operator class""" + + keylist = ['schema', 'name', 'index_method'] + single_extern_file = True + catalog = 'pg_opclass' + + def __init__(self, name, schema, index_method, description, owner, + family, type, default=None, storage=None, oid=None): + """Initialize the operator class + + :param name: operator name (from opcname) + :param schema: schema name (from opcnamespace) + :param index_method: index access method (from amname via opcmethod) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via opcowner) + :param family: operator family (from opfname via opcfamily) + :param type: data type indexed (from opcintype) + :param default: default class for this type? (from opcdefault) + :param storage: type of data stored (from opckeytype) + """ + super(OperatorClass, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.index_method = index_method + self.family = family + (sch, typ) = split_schema_obj(type) + self.type = typ if self.schema == sch else type + self.default = default + self.storage = storage if storage != '-' else None + self.operators = {} + self.functions = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, opcname AS name, rolname AS owner, + amname AS index_method, opfname AS family, + opcintype::regtype AS type, opcdefault AS default, + opckeytype::regtype AS storage, + obj_description(o.oid, 'pg_opclass') AS description, o.oid + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_roles r ON (r.oid = opcowner) + JOIN pg_opfamily f ON (opcfamily = f.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname""" + + @staticmethod + def opquery(): + return """ + SELECT nspname AS schema, opcname AS name, amname AS index_method, + amopstrategy AS strategy, amopopr::regoperator AS operator + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid), pg_amop ao, + pg_depend + WHERE refclassid = 'pg_opclass'::regclass + AND classid = 'pg_amop'::regclass AND objid = ao.oid + AND refobjid = o.oid + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname, amopstrategy""" + + @staticmethod + def prquery(): + return """ + SELECT nspname AS schema, opcname AS name, amname AS index_method, + amprocnum AS support, amproc::regprocedure AS function + FROM pg_opclass o JOIN pg_am a ON (opcmethod = a.oid) + JOIN pg_namespace n ON (opcnamespace = n.oid), pg_amproc ap, + pg_depend + WHERE refclassid = 'pg_opclass'::regclass + AND classid = 'pg_amproc'::regclass AND objid = ap.oid + AND refobjid = o.oid + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opclass'::regclass) + ORDER BY nspname, opcname, amname, amprocnum""" + + @staticmethod + def from_map(name, schema, index_method, inobj): + """Initialize an operator class instance from a YAML map + + :param name: operator class name + :param name: schema name + :param index_method: index method + :param inobj: YAML map of the operator class + :return: operator class instance + """ + obj = OperatorClass( + name, schema.name, index_method, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('family', None), + inobj.pop('type', None), inobj.pop('default', False), + inobj.pop('storage', None)) + if 'operators' in inobj: + obj.operators = inobj.get('operators') + if 'functions' in inobj: + obj.functions = inobj.get('functions') + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "OPERATOR CLASS" + + def extern_key(self): + """Return the key to be used in external maps for this operator + + :return: string + """ + return '%s %s using %s' % (self.objtype.lower(), self.name, + self.index_method) + + def identifier(self): + """Return a full identifier for an operator class + + :return: string + """ + return "%s USING %s" % (self.qualname(), self.index_method) + + def to_map(self, db, no_owner): + """Convert operator class to a YAML-suitable format + + :return: dictionary + """ + dct = super(OperatorClass, self).to_map(db, no_owner) + if self.storage is None: + del dct['storage'] + if not self.default: + del dct['default'] + if self.name == self.family: + del dct['family'] + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the operator class + + :return: SQL statements + """ + dflt = '' + if self.default: + dflt = "DEFAULT " + clauses = [] + for (strat, oper) in list(self.operators.items()): + clauses.append("OPERATOR %d %s" % (strat, oper)) + for (supp, func) in list(self.functions.items()): + clauses.append("FUNCTION %d %s" % (supp, func)) + if self.storage is not None: + clauses.append("STORAGE %s" % self.storage) + return ["CREATE OPERATOR CLASS %s\n %sFOR TYPE %s USING %s " + "AS\n %s" % ( + self.qualname(), dflt, + self.qualname(self.schema, self.type), self.index_method, + ',\n ' .join(clauses))] + + def get_implied_deps(self, db): + deps = super(OperatorClass, self).get_implied_deps(db) + + type = db.types.find(self.type) + if type: + deps.add(type) + + if self.storage is not None: + type = db.types.find(self.storage) + if type: + deps.add(type) + + for f in self.functions.values(): + f = db.functions.find(*split_func_args(f)) + if f is not None: + deps.add(f) + + for f in self.operators.values(): + f = db.operators.find(f) + if f is not None: + deps.add(f) + + if self.family is not None: + f = db.operfams.find(self.schema, self.family, self.index_method) + if f is not None: + deps.add(f) + + return deps + + +class OperatorClassDict(DbObjectDict): + "The collection of operator classes in a database" + + cls = OperatorClass + + def _from_catalog(self): + """Initialize the dictionary of operator classes from the catalogs""" + for opclass in self.fetch(): + self[opclass.key()] = opclass + opers = self.dbconn.fetchall(self.cls.opquery()) + self.dbconn.rollback() + for opdata in opers: + sch = opdata["schema"] + opc = opdata["name"] + idx = opdata["index_method"] + strat = opdata["strategy"] + oper = opdata["operator"] + opcls = self[(sch, opc, idx)] + opcls.operators.update({strat: oper}) + funcs = self.dbconn.fetchall(self.cls.prquery()) + self.dbconn.rollback() + for oprdata in funcs: + sch = oprdata["schema"] + opc = oprdata["name"] + idx = oprdata["index_method"] + supp = oprdata["support"] + func = oprdata["function"] + opcls = self[(sch, opc, idx)] + opcls.functions.update({supp: func}) + + def from_map(self, schema, inopcls): + """Initialize the dictionary of operator classes from the input map + + :param schema: schema owning the operator classes + :param inopcls: YAML map defining the operator classes + """ + for key in inopcls: + if not key.startswith('operator class ') or ' using ' not in key: + raise KeyError("Unrecognized object type: %s" % key) + pos = key.rfind(' using ') + opc = key[15:pos] # 15 = len('operator class ') + idx = key[pos + 7:] # 7 = len(' using ') + inobj = inopcls[key] + self[(schema.name, opc, idx)] = OperatorClass.from_map( + opc, schema, idx, inobj) diff --git a/pyrseas/dbobject/operfamily.py b/pyrseas/dbobject/operfamily.py new file mode 100644 index 0000000..e9e3277 --- /dev/null +++ b/pyrseas/dbobject/operfamily.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.operfamily + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: OperatorFamily derived from + DbSchemaObject and OperatorFamilyDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_schema_obj + + +class OperatorFamily(DbSchemaObject): + """An operator family""" + + keylist = ['schema', 'name', 'index_method'] + single_extern_file = True + catalog = 'pg_opfamily' + + def __init__(self, name, schema, index_method, description, owner, + oid=None): + """Initialize the operator family + + :param name: operator name (from opfname) + :param schema: schema name (from opfnamespace) + :param index_method: index access method (from amname via opfmethod) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via opfowner) + """ + super(OperatorFamily, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.index_method = index_method + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, opfname AS name, rolname AS owner, + amname AS index_method, + obj_description(o.oid, 'pg_opfamily') AS description, o.oid + FROM pg_opfamily o JOIN pg_roles r ON (r.oid = opfowner) + JOIN pg_am a ON (opfmethod = a.oid) + JOIN pg_namespace n ON (opfnamespace = n.oid) + WHERE (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND o.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_opfamily'::regclass) + ORDER BY opfnamespace, opfname, amname""" + + @staticmethod + def from_map(name, schema, index_method, inobj): + """Initialize an operator family instance from a YAML map + + :param name: operator family name + :param name: schema name + :param index_method: index method + :param inobj: YAML map of the operator family + :return: operator family instance + """ + obj = OperatorFamily( + name, schema.name, index_method, inobj.pop('description', None), + inobj.pop('owner', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "OPERATOR FAMILY" + + def extern_key(self): + """Return the key to be used in external maps for the operator family + + :return: string + """ + return '%s %s using %s' % (self.objtype.lower(), self.name, + self.index_method) + + def identifier(self): + """Return a full identifier for an operator family object + + :return: string + """ + return "%s USING %s" % (self.qualname(), self.index_method) + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the operator family + + :return: SQL statements + """ + return ["CREATE OPERATOR FAMILY %s USING %s" % ( + self.qualname(), self.index_method)] + + +class OperatorFamilyDict(DbObjectDict): + "The collection of operator families in a database" + + cls = OperatorFamily + + def from_map(self, schema, inopfams): + """Initialize the dict of operator families by converting the input map + + :param schema: schema owning the operators + :param inopfams: YAML map defining the operator families + """ + for key in inopfams: + if not key.startswith('operator family ') or ' using ' not in key: + raise KeyError("Unrecognized object type: %s" % key) + pos = key.rfind(' using ') + opf = key[16:pos] # 16 = len('operator family ') + idx = key[pos + 7:] # 7 = len(' using ') + inobj = inopfams[key] + self[(schema.name, opf, idx)] = OperatorFamily.from_map( + opf, schema, idx, inobj) + + def find(self, sch, obj, meth): + schema, name = split_schema_obj(obj, sch) + return self.get((schema, name, meth)) diff --git a/pyrseas/dbobject/privileges.py b/pyrseas/dbobject/privileges.py new file mode 100644 index 0000000..4c9b48c --- /dev/null +++ b/pyrseas/dbobject/privileges.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.privileges + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines functions for dealing with access privileges. +""" + +PRIVCODES = {'a': 'insert', 'r': 'select', 'w': 'update', 'd': 'delete', + 'D': 'truncate', 'x': 'references', 't': 'trigger', + 'X': 'execute', 'U': 'usage', 'C': 'create'} +PRIVILEGES = dict((v, k) for k, v in list(PRIVCODES.items())) + + +def _split_privs(privspec): + """Split the aclitem into three parts + + :param privspec: privilege specification (aclitem) + :return: tuple with grantee, privilege codes and granto + """ + (usr, prvgrant) = privspec.split('=') + if usr == '': + usr = 'PUBLIC' + (privcodes, grantor) = prvgrant.split('/') + return (usr, privcodes, grantor) + + +def _expand_priv_lists(obj, privcodes, subobj): + """Convert privilege code strings to expanded lists + + :param obj: the object on which the privilege is granted + :param privcodes: string of privilege codes + :param subobj: sub-object name (e.g., column name) + :return: tuple of lists with decoded privileges + """ + privs = [] + wgo = [] + if privcodes == obj.allprivs and len(obj.allprivs) > 1: + privs = ['ALL'] + else: + if subobj: + subobj = ' (%s)' % subobj + for code in sorted(PRIVCODES.keys()): + if code in privcodes: + priv = PRIVCODES[code].upper() + subobj + if code + '*' in privcodes: + wgo.append(priv) + else: + privs.append(priv) + return (privs, wgo) + + +def privileges_to_map(privspec, allprivs, owner): + """Map a set of privileges in PostgreSQL format to YAML-suitable format + + :param privspec: privilege specification + :param allprivs: privilege list equal to ALL + :param owner: object owner + :return: dictionary + + Access privileges are specified as aclitem's as follows: + =/. The grantee and grantor are user + names. The privlist is a set of single letter codes, each letter + optionally followed by an asterisk to indicate WITH GRANT OPTION. + """ + (usr, privcodes, grantor) = _split_privs(privspec) + privs = [] + if privcodes == allprivs and len(allprivs) > 1: + privs = ['all'] + else: + for code in sorted(PRIVCODES.keys()): + if code in privcodes: + priv = PRIVCODES[code] + if code + '*' in privcodes: + priv = {priv: {'grantable': True}} + privs.append(priv) + if owner and grantor != owner: + privs = {'privs': privs, 'grantor': grantor} + return {usr: privs} + + +def privileges_from_map(privlist, allprivs, owner): + """Map privileges from YAML-suitable format to an internal list + + :param privspec: privilege specification + :param allprivs: privilege list equal to ALL + :param owner: object owner + :return: list + """ + retlist = [] + for priv in privlist: + usr = list(priv.keys())[0] + privs = priv[usr] + grantor = owner + if 'grantor' in privs: + grantor = privs['grantor'] + privs = privs['privs'] + if usr == 'PUBLIC': + usr = '' + prvcodes = '' + if privs == ['all']: + prvcodes = allprivs + else: + for prv in privs: + if isinstance(prv, dict): + key = list(prv.keys())[0] + else: + key = prv + if key in PRIVILEGES: + prvcodes += PRIVILEGES[key] + if isinstance(prv, dict) and isinstance(prv[key], dict) and \ + 'grantable' in prv[key] and prv[key]['grantable']: + prvcodes += '*' + retlist.append("%s=%s/%s" % (usr, prvcodes, grantor)) + return retlist + + +def add_grant(obj, privspec, subobj=''): + """Return GRANT statements on the object based on the privilege spec + + :param obj: the object on which the privilege is granted + :param privspec: the privilege specification (aclitem) + :param subobj: sub-object name (e.g., column name) + :return: list of GRANT statements + """ + (usr, privcodes, grantor) = _split_privs(privspec) + (privs, wgo) = _expand_priv_lists(obj, privcodes, subobj) + objtype = obj.objtype + if hasattr(obj, 'privobjtype'): + objtype = obj.privobjtype + stmts = [] + if privs: + stmts.append("GRANT %s ON %s %s TO %s" % ( + ', '.join(privs), objtype, obj.identifier(), usr)) + if wgo: + stmts.append("GRANT %s ON %s %s TO %s WITH GRANT OPTION" % ( + ', '.join(wgo), objtype, obj.identifier(), usr)) + return stmts + + +def add_revoke(obj, privspec, subobj=''): + """Return REVOKE statements on the object based on the privilege spec + + :param obj: the object on which the privilege is to be revoked + :param privspec: the privilege specification (aclitem) + :param subobj: sub-object name (e.g., column name) + :return: list of REVOKE statements + """ + (usr, privcodes, grantor) = _split_privs(privspec) + (privs, wgo) = _expand_priv_lists(obj, privcodes, subobj) + objtype = obj.objtype + if hasattr(obj, 'privobjtype'): + objtype = obj.privobjtype + stmts = [] + if wgo: + stmts.append("REVOKE %s ON %s %s FROM %s" % ( + ', '.join(wgo), objtype, obj.identifier(), usr)) + if privs: + stmts.append("REVOKE %s ON %s %s FROM %s" % ( + ', '.join(privs), objtype, obj.identifier(), usr)) + return stmts + + +def diff_privs(currobj, currlist, newobj, newlist, subobj=''): + """Return GRANT or REVOKE statements to adjust object privileges + + :param currobj: current object + :param currlist: list of current privileges + :param newobj: new object + :param newlist: list of new privileges + :param subobj: sub-object (e.g., column name) + :return: list of GRANT and REVOKE statements + """ + def rejoin(privdict, usr, grantor): + return '%s=%s/%s' % ('' if usr == 'PUBLIC' else usr, + privdict[(usr, grantor)], grantor) + + stmts = [] + currprivs = {} + newprivs = {} + for privspec in currlist: + if privspec: + (usr, privcodes, grantor) = _split_privs(privspec) + currprivs[(usr, grantor)] = privcodes + for privspec in newlist: + if privspec: + (usr, privcodes, grantor) = _split_privs(privspec) + newprivs[(usr, grantor)] = privcodes + for (usr, gtor) in currprivs: + if (usr, gtor) not in newprivs: + stmts.append(add_revoke(currobj, rejoin(currprivs, usr, gtor), + subobj)) + for (usr, gtor) in newprivs: + if (usr, gtor) not in currprivs: + stmts.append(add_grant(newobj, rejoin(newprivs, usr, gtor), + subobj)) + else: + if sorted(currprivs[(usr, gtor)]) != sorted(newprivs[(usr, gtor)]): + stmts.append(add_revoke(currobj, rejoin(currprivs, usr, gtor), + subobj)) + stmts.append(add_grant(newobj, rejoin(newprivs, usr, gtor), + subobj)) + return stmts diff --git a/pyrseas/dbobject/rule.py b/pyrseas/dbobject/rule.py new file mode 100644 index 0000000..9f50b60 --- /dev/null +++ b/pyrseas/dbobject/rule.py @@ -0,0 +1,143 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.rule + ~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Rule and RuleDict, derived from + DbSchemaObject and DbObjectDict, respectively. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, split_schema_obj + + +class Rule(DbSchemaObject): + """A rewrite rule definition""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_rewrite' + + def __init__(self, name, schema, table, description, event, instead=False, + actions=None, condition=None, definition=None, + oid=None): + """Initialize the rewrite rule + + :param name: rule name (from rulename) + :param schema: schema name (from nspname via relnamespace/ev_class) + :param table: table name (from relname via ev_class) + :param description: comment text (from obj_description()) + :param event: event type (from ev_type) + :param instead: is it an INSTEAD rule? (from is_instead) + :param actions: rule actions (from ev_action via definition) + :param condition: qualifying condition (from ev_qual via definition) + :param definition: "raw" definition (from pg_get_ruledef) + """ + super(Rule, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = table + self.description = description + self.event = event + self.instead = instead + if definition is not None: + assert actions is None + assert condition is None + do_loc = definition.index(' DO ') + if 'WHERE' in definition: + self.condition = definition[ + definition.index(' WHERE ') + 7:do_loc] + if instead: + do_loc += 8 + self.actions = definition[do_loc + 4:-1] + else: + self.actions = actions + self.condition = condition + self.oid = oid + + @staticmethod + def query(dbversion=None): + """Returns query to fetch Rule instances from the catalogs""" + return """ + SELECT nspname AS schema, relname AS table, rulename AS name, + split_part('select,update,insert,delete', ',', + ev_type::int - 48) AS event, is_instead AS instead, + pg_get_ruledef(r.oid) AS definition, + obj_description(r.oid, 'pg_rewrite') AS description, r.oid + FROM pg_rewrite r JOIN pg_class c ON (ev_class = c.oid) + JOIN pg_namespace n ON (relnamespace = n.oid) + WHERE relkind = 'r' + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + ORDER BY nspname, relname, rulename""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a Rule instance from a YAML map + + :param name: rule name + :param table: map of the table associated with the rule + :param inobj: YAML map of the rule + :return: Rule instance + """ + obj = Rule( + name, table.schema, table.name, inobj.pop('description', None), + inobj.get('event'), inobj.pop('instead', False), + inobj.pop('actions', None), inobj.pop('condition', None)) + obj.set_oldname(inobj) + return obj + + def identifier(self): + """Return a full identifier for a rule object + + :return: string + """ + return "%s ON %s" % (quote_id(self.name), self._table.qualname()) + + def to_map(self, db): + """Convert rule to a YAML-suitable format + + :return: dictionary + """ + dct = super(Rule, self).to_map(db) + if not self.instead: + dct.pop('instead') + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the rule + + :return: SQL statements + """ + where = instead = '' + if self.condition is not None: + where = ' WHERE %s' % self.condition + if self.instead: + instead = 'INSTEAD ' + return ["CREATE RULE %s AS ON %s\n TO %s%s\n DO %s%s" % ( + quote_id(self.name), self.event.upper(), + self._table.qualname(), where, instead, self.actions)] + + def get_implied_deps(self, db): + """Return set of implicit dependencies + + :param db: Database.Dicts object + :return: set of dependencies + """ + deps = super(Rule, self).get_implied_deps(db) + deps.add(db.tables[split_schema_obj(self.table, self.schema)]) + return deps + + +class RuleDict(DbObjectDict): + "The collection of rewrite rules in a database." + + cls = Rule + + def from_map(self, table, inmap): + """Initialize the dictionary of rules by examining the input map + + :param table: the input YAML map of the associated table + :param inmap: the input YAML map defining the rules + """ + for rul in inmap: + inobj = inmap[rul] + self[(table.schema, table.name, rul)] = Rule.from_map( + rul, table, inobj) diff --git a/pyrseas/dbobject/schema.py b/pyrseas/dbobject/schema.py new file mode 100644 index 0000000..9f5d767 --- /dev/null +++ b/pyrseas/dbobject/schema.py @@ -0,0 +1,371 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.schema + ~~~~~~~~~~~~~~~~~~~~~~~ + + This defines two classes, Schema and SchemaDict, derived from + DbObject and DbObjectDict, respectively. +""" +import os + +from pyrseas.yamlutil import yamldump +from . import DbObjectDict, DbObject +from . import quote_id, commentable, ownable, grantable +from .dbtype import BaseType, Composite, Domain, Enum, Range +from .table import Table, Sequence +from .view import View, MaterializedView + + +class Schema(DbObject): + """A database schema definition, i.e., a named collection of tables, + views, triggers and other schema objects.""" + + keylist = ['name'] + catalog = 'pg_namespace' + + @property + def allprivs(self): + return 'UC' + + def __init__(self, name, description=None, owner=None, privileges=[], + oldname=None, oid=None): + """Initialize the schema + + :param name: schema name (from nspname) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via nspowner) + :param privileges: access privileges (from nspacl) + :param oldname: previous name of schema + """ + super(Schema, self).__init__(name, description) + self._init_own_privs(owner, privileges) + self.oldname = None + + @staticmethod + def query(dbversion=None): + return r""" + SELECT nspname AS name, rolname AS owner, + array_to_string(nspacl, ',') AS privileges, + obj_description(n.oid, 'pg_namespace') AS description, n.oid + FROM pg_namespace n + JOIN pg_roles r ON (r.oid = nspowner) + WHERE nspname NOT IN ('information_schema', 'pg_toast') + AND nspname NOT LIKE 'pg_temp\_%' + AND nspname NOT LIKE 'pg_toast_temp\_%' + AND n.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_namespace'::regclass) + ORDER BY nspname""" + + @staticmethod + def from_map(name, inobj): + """Initialize a schema instance from a YAML map + + :param name: schema name + :param inobj: YAML map of the schema + :return: schema instance + """ + obj = Schema( + name, inobj.pop('description', None), inobj.pop('owner', None), + inobj.pop('privileges', [])) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + def extern_dir(self, root='.'): + """Return the path to a directory to hold the schema objects. + + :return: directory path + """ + (dir, ext) = os.path.splitext(os.path.join(root, + self.extern_filename())) + return dir + + def to_map(self, db, dbschemas, opts): + """Convert tables, etc., dictionaries to a YAML-suitable format + + :param dbschemas: dictionary of schemas + :param opts: options to include/exclude schemas/tables, etc. + :return: dictionary + """ + if self.name == 'pyrseas': + return {} + no_owner = opts.no_owner + no_privs = opts.no_privs + schbase = {} if no_owner else {'owner': self.owner} + if not no_privs: + schbase.update({'privileges': self.map_privs()}) + if self.description is not None: + schbase.update(description=self.description) + + schobjs = [] + seltbls = getattr(opts, 'tables', []) + if hasattr(self, 'tables'): + for objkey in self.tables: + if not seltbls or objkey in seltbls: + obj = self.tables[objkey] + schobjs.append((obj, obj.to_map(db, dbschemas, opts))) + + def mapper(objtypes): + if hasattr(self, objtypes): + schemadict = getattr(self, objtypes) + for objkey in schemadict: + if objtypes == 'sequences' or ( + not seltbls or objkey in seltbls): + obj = schemadict[objkey] + schobjs.append((obj, obj.to_map(db, opts))) + + for objtypes in ['ftables', 'sequences', 'views', 'matviews']: + mapper(objtypes) + + def mapper2(objtypes, privs=False): + if hasattr(self, objtypes): + schemadict = getattr(self, objtypes) + for objkey in schemadict: + obj = schemadict[objkey] + if privs: + dct = obj.to_map(db, no_owner, no_privs) + else: + dct = obj.to_map(db, no_owner) + schobjs.append((obj, dct)) + + if hasattr(opts, 'tables') and not opts.tables or \ + not hasattr(opts, 'tables'): + for objtypes in ('conversions', 'collations', 'operators', + 'operclasses', 'operfams', 'tsconfigs', + 'tsdicts', 'tsparsers', 'tstempls'): + mapper2(objtypes) + for objtypes in ('types', 'domains'): + mapper2(objtypes, True) + if hasattr(self, 'functions'): + for objkey in self.functions: + obj = self.functions[objkey] + schobjs.append((obj, obj.to_map(db, no_owner, no_privs))) + + # special case for pg_catalog schema + if self.name == 'pg_catalog' and not schobjs: + return {} + + if hasattr(self, 'datacopy') and self.datacopy: + dir = self.extern_dir(opts.data_dir) + if not os.path.exists(dir): + os.mkdir(dir) + for tbl in self.datacopy: + self.tables[tbl].data_export(dbschemas.dbconn, dir) + dbschemas.dbconn.commit() + + if opts.multiple_files: + dir = self.extern_dir(opts.metadata_dir) + if not os.path.exists(dir): + os.mkdir(dir) + filemap = {} + for obj, objmap in schobjs: + if objmap is not None: + extkey = obj.extern_key() + filepath = os.path.join(dir, obj.extern_filename()) + with open(filepath, 'a') as f: + f.write(yamldump({extkey: objmap})) + outobj = {extkey: + os.path.relpath(filepath, opts.metadata_dir)} + filemap.update(outobj) + # always write the schema YAML file + filepath = self.extern_filename() + extkey = self.extern_key() + with open(os.path.join(opts.metadata_dir, filepath), 'a') as f: + f.write(yamldump({extkey: schbase})) + filemap.update(schema=filepath) + return {extkey: filemap} + + schmap = dict((obj.extern_key(), objmap) for obj, objmap in schobjs + if objmap is not None) + schmap.update(schbase) + return {self.extern_key(): schmap} + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the schema + + :return: SQL statements + """ + # special case for --revert + if self.name == 'pg_catalog': + return [] + return ["CREATE SCHEMA %s" % quote_id(self.name)] + + def data_import(self, opts): + """Generate SQL to import data from the tables in this schema + + :param opts: options to include/exclude schemas/tables, etc. + :return: list of SQL statements + """ + stmts = [] + if hasattr(self, 'datacopy') and self.datacopy: + dir = self.extern_dir(opts.data_dir) + for tbl in self.datacopy: + stmts.append(self.tables[tbl].data_import(dir)) + return stmts + + def drop(self): + if self.name not in ('public', 'pg_catalog'): + return super(Schema, self).drop() + else: + return [] + + +PREFIXES = {'domain ': 'types', 'type': 'types', 'table ': 'tables', + 'view ': 'tables', 'sequence ': 'tables', + 'materialized view ': 'tables', + 'function ': 'functions', 'aggregate ': 'functions', + 'operator family ': 'operfams', 'operator class ': 'operclasses', + 'conversion ': 'conversions', 'text search dictionary ': 'tsdicts', + 'text search template ': 'tstempls', + 'text search parser ': 'tsparsers', + 'text search configuration ': 'tsconfigs', + 'foreign table ': 'ftables', 'collation ': 'collations'} +SCHOBJS1 = ['types', 'tables', 'ftables'] +SCHOBJS2 = ['collations', 'conversions', 'functions', 'operators', + 'operclasses', 'operfams', 'tsconfigs', 'tsdicts', 'tsparsers', + 'tstempls'] + + +class SchemaDict(DbObjectDict): + "The collection of schemas in a database. Minimally, the 'public' schema." + + cls = Schema + + def from_map(self, inmap, newdb): + """Initialize the dictionary of schemas by converting the input map + + :param inmap: the input YAML map defining the schemas + :param newdb: collection of dictionaries defining the database + + Starts the recursive analysis of the input map and + construction of the internal collection of dictionaries + describing the database objects. + """ + for key in inmap: + (objtype, spc, sch) = key.partition(' ') + if spc != ' ' or objtype != 'schema': + raise KeyError("Unrecognized object type: %s" % key) + inobj = inmap[key] + schema = self[sch] = Schema.from_map(sch, inobj) + objdict = {} + for key in sorted(inobj.keys()): + mapped = False + for prefix in PREFIXES: + if key.startswith(prefix): + otype = PREFIXES[prefix] + if otype not in objdict: + objdict[otype] = {} + objdict[otype].update({key: inobj[key]}) + mapped = True + break + # Needs separate processing because it overlaps + # operator classes and operator families + if not mapped and key.startswith('operator '): + otype = 'operators' + if otype not in objdict: + objdict[otype] = {} + objdict[otype].update({key: inobj[key]}) + mapped = True + elif key == 'oldname': + inobj.pop(key) + mapped = True + if not mapped and key != 'schema': + raise KeyError("Expected typed object, found '%s'" % key) + + for objtype in SCHOBJS1: + if objtype in objdict: + subobjs = getattr(newdb, objtype) + subobjs.from_map(schema, objdict[objtype], newdb) + for objtype in SCHOBJS2: + if objtype in objdict: + subobjs = getattr(newdb, objtype) + subobjs.from_map(schema, objdict[objtype]) + + def link_refs(self, db, datacopy): + """Connect various schema objects to their respective schemas + + :param db: dictionary of dictionaries of all objects + :param datacopy: dictionary of data copying info + """ + def link_one(targdict, objtype, objkeys, subtype=None): + schema = self[objkeys[0]] + if subtype is None: + subtype = objtype + if not hasattr(schema, subtype): + setattr(schema, subtype, {}) + objdict = getattr(schema, subtype) + key = objkeys[1] if len(objkeys) == 2 else objkeys[1:] + objdict.update({key: targdict[objkeys]}) + + targ = db.types + for keys in targ: + dbtype = targ[keys] + if isinstance(dbtype, Domain): + link_one(targ, 'types', keys, 'domains') + elif isinstance(dbtype, (Enum, Composite, BaseType, Range)): + link_one(targ, 'types', keys) + targ = db.tables + for keys in targ: + table = targ[keys] + type_ = 'tables' + if isinstance(table, Table): + link_one(targ, type_, keys) + elif isinstance(table, Sequence): + link_one(targ, type_, keys, 'sequences') + elif isinstance(table, MaterializedView): + link_one(targ, type_, keys, 'matviews') + elif isinstance(table, View): + link_one(targ, type_, keys, 'views') + targ = db.functions + for keys in targ: + link_one(targ, 'functions', keys) + for objtype in ['operators', 'operclasses', 'operfams', 'conversions', + 'tsconfigs', 'tsdicts', 'tsparsers', 'tstempls', + 'ftables', 'collations']: + targ = getattr(db, objtype) + for keys in targ: + link_one(targ, objtype, keys) + for key in datacopy: + if not key.startswith('schema '): + raise KeyError("Unrecognized object type: %s" % key) + sch = key[7:] + if sch not in self: + continue + schema = self[sch] + if not hasattr(schema, 'datacopy'): + schema.datacopy = [] + for tbl in datacopy[key]: + if hasattr(schema, 'tables') and tbl in schema.tables: + schema.datacopy.append(tbl) + + def to_map(self, db, opts): + """Convert the schema dictionary to a regular dictionary + + :param opts: options to include/exclude schemas/tables, etc. + :return: dictionary + + Invokes the `to_map` method of each schema to construct a + dictionary of schemas. + """ + schemas = {} + selschs = getattr(opts, 'schemas', []) + for sch in self: + if not selschs or sch in selschs: + if hasattr(opts, 'excl_schemas') and opts.excl_schemas \ + and sch in opts.excl_schemas: + continue + schemas.update(self[sch].to_map(db, self, opts)) + + return schemas + + def data_import(self, opts): + """Iterate over schemas with tables to be imported + + :param opts: options to include/exclude schemas/tables, etc. + :return: list of SQL statements + """ + return [self[sch].data_import(opts) for sch in self] diff --git a/pyrseas/dbobject/table.py b/pyrseas/dbobject/table.py new file mode 100644 index 0000000..66a49d9 --- /dev/null +++ b/pyrseas/dbobject/table.py @@ -0,0 +1,985 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.table + ~~~~~~~~~~~~~~~~~~~~~~ + + This module defines four classes: DbClass derived from + DbSchemaObject, Sequence and Table derived from DbClass, and + ClassDict derived from DbObjectDict. +""" + +import copy +import re +import os +import sys + +from . import DbObjectDict, DbSchemaObject, split_schema_obj +from . import quote_id, commentable, ownable, grantable +from .constraint import CheckConstraint, PrimaryKey +from .constraint import ForeignKey, UniqueConstraint +from .privileges import add_grant + +MAX_BIGINT = 9223372036854775807 + + +def seq_max_value(seq): + if seq.max_value is None or seq.max_value == MAX_BIGINT: + return " NO MAXVALUE" + return " MAXVALUE %d" % seq.max_value + + +def seq_min_value(seq): + if seq.min_value is None or seq.min_value == 1: + return " NO MINVALUE" + return " MINVALUE %d" % seq.min_value + + +class DbClass(DbSchemaObject): + """A table, sequence or view + + The `pg_class` catalog also includes Postgres indexes, but for now, + indexes have not been implemented as part of the `DbClass` hierarchy. + """ + + keylist = ['schema', 'name'] + catalog = 'pg_class' + + def __init__(self, name, schema, description, owner, privileges): + """Initialize the relation "class" + + :param name: relation name (from relname) + :param schema: schema name (from relnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via relowner) + :param privileges: access privileges (from relacl) + """ + super(DbClass, self).__init__(name, schema, description) + self._init_own_privs(owner, privileges) + + +class Sequence(DbClass): + "A sequence generator definition" + + def __init__(self, name, schema, description, owner, privileges, + start_value=1, increment_by=1, max_value=MAX_BIGINT, + min_value=1, cache_value=1, data_type='bigint', + owner_table=None, owner_column=None, + oid=None): + """Initialize the sequence + + :param name-privileges: see DbClass.__init__ params + :param start_value: start value (from start_value) + :param max_value: maximum value (from max_value) + :param increment_by: value to add (from increment_by) + :param min_value: minimum value (from min_value) + :param cache_value: cache value (from cache_value/cache_size) + :param data_type: data type (from data_type) + :param owner_table: owner table + :param owner_column: owner column + """ + super(Sequence, self).__init__(name, schema, description, owner, + privileges) + self.start_value = start_value + self.increment_by = increment_by + self.max_value = max_value + self.min_value = min_value + self.cache_value = cache_value + self.data_type = data_type + self.owner_table = owner_table + self.owner_column = owner_column + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'S' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + AND c.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_class'::regclass) + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a sequence instance from a YAML map + + :param name: sequence name + :param name: schema map + :param inobj: YAML map of the sequence + :return: sequence instance + """ + obj = Sequence( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('start_value', 1), inobj.pop('increment_by', 1), + inobj.pop('max_value', MAX_BIGINT), inobj.pop('min_value', 1), + inobj.pop('cache_value', 1), inobj.pop('data_type', 'bigint'), + inobj.pop('owner_table', None), inobj.pop('owner_column', None)) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def allprivs(self): + return 'rwU' + + def get_attrs(self, dbconn): + """Get the attributes for the sequence + + :param dbconn: a DbConnection object + """ + query = """SELECT start_value, increment_by, max_value, min_value, + cache_size AS cache_value, data_type + FROM pg_sequences + WHERE schemaname = '%s' + AND sequencename = '%s'""" % (self.schema, self.name) + data = dbconn.fetchone(query) + + for key, val in list(data.items()): + setattr(self, key, val) + + def get_dependent_table(self, dbconn): + """Get the table and column name that uses or owns the sequence + + :param dbconn: a DbConnection object + """ + + def split_table(obj, sch): + schema = sch + tbl = obj + quoted = '"%s".' % schema + if obj.startswith(schema + '.'): + tbl = obj[len(schema) + 1:] + elif obj.startswith(quoted): + tbl = obj[len(quoted):] + elif sch is None: + raise ValueError("Invalid schema.table: %s" % obj) + if tbl[0] == '"' and tbl[-1:] == '"': + tbl = tbl[1:-1] + return tbl + + data = dbconn.fetchone( + """SELECT refobjid::regclass, refobjsubid + FROM pg_depend + WHERE objid = '%s'::regclass + AND refclassid = 'pg_class'::regclass""" % self.identifier()) + if data: + self.owner_table = split_table(data["refobjid"], self.schema) + self.owner_column = data["refobjsubid"] + return + data = dbconn.fetchone( + """SELECT adrelid::regclass AS regclass + FROM pg_attrdef a JOIN pg_depend ON (a.oid = objid) + WHERE refobjid = '%s'::regclass + AND classid = 'pg_attrdef'::regclass""" % self.qualname()) + if data: + self.dependent_table = split_table(data["regclass"], self.schema) + + def to_map(self, db, opts): + """Convert a sequence definition to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'tables') and opts.tables and \ + (self.name not in opts.tables and + self.owner_table is None or + self.owner_table not in opts.tables) or ( + hasattr(opts, 'excl_tables') and opts.excl_tables and + self.name in opts.excl_tables): + return None + dct = super(Sequence, self).to_map(db, opts.no_owner, opts.no_privs) + if self.owner_table is None and self.owner_column is None: + dct.pop('owner_table') + dct.pop('owner_column') + dct.pop('dependent_table', None) + if self.data_type == 'bigint': + dct.pop('data_type') + for key, val in list(dct.items()): + if key == 'max_value' and val == MAX_BIGINT: + dct[key] = None + elif key == 'min_value' and val == 1: + dct[key] = None + elif key == 'privileges': + dct[key] = val + else: + if isinstance(val, int): + dct[key] = int(val) + else: + dct[key] = str(val) + + return dct + + def _common_create_spec(self): + return """ START WITH %d + INCREMENT BY %d + %s + %s + CACHE %d""" % (self.start_value, self.increment_by, seq_min_value(self), + seq_max_value(self), self.cache_value) + + @commentable + @grantable + @ownable + def create(self, dbversion=None): + """Return a SQL statement to CREATE the sequence + + :return: SQL statements + """ + mod = "" + if self.data_type != 'bigint': + mod = "\n AS %s" % self.data_type + if hasattr(self, '_owner_col'): + if self._owner_col.identity is not None: + return [] + return ["CREATE SEQUENCE %s%s%s" % (self.qualname(), mod, + self._common_create_spec())] + + def add_inline(self): + """Return statement to create the sequence inline as part of a + GENERATED AS IDENTITY clause + """ + return "SEQUENCE NAME %s%s" % (self.qualname(), + self._common_create_spec()) + + def add_owner(self): + """Return statement to ALTER the sequence to indicate its owner table + + :return: SQL statement + """ + stmts = [] + stmts.append("ALTER SEQUENCE %s OWNED BY %s.%s" % ( + self.qualname(), self.qualname(self.schema, self.owner_table), + quote_id(self.owner_column))) + return stmts + + def alter(self, inseq, no_owner=False): + """Generate SQL to transform an existing sequence + + :param inseq: a YAML map defining the new sequence + :return: list of SQL statements + + Compares the sequence to an input sequence and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + stmt = "" + if self.start_value != inseq.start_value: + stmt += " START WITH %d" % inseq.start_value + if self.increment_by != inseq.increment_by: + stmt += " INCREMENT BY %d" % inseq.increment_by + minval = self.min_value + if minval == 1: + minval = None + if minval != inseq.min_value: + stmt += seq_min_value(inseq) + maxval = self.max_value + if maxval == MAX_BIGINT: + maxval = None + if maxval != inseq.max_value: + stmt += seq_max_value(inseq) + if self.cache_value != inseq.cache_value: + stmt += " CACHE %d" % inseq.cache_value + if stmt: + stmts.append("ALTER SEQUENCE %s" % self.qualname() + stmt) + + if inseq.owner_column is not None and inseq.owner_table is None: + raise ValueError("Sequence '%s' incomplete specification: " + "owner_column but no owner_table") + if inseq.owner_table is not None: + if inseq.owner_column is None: + raise ValueError("Sequence '%s' incomplete specification: " + "owner_table but no owner_column") + if self.owner_table is None and self.owner_column is None: + stmts.append(inseq.add_owner()) + + stmts.append(super(Sequence, self).alter(inseq, no_owner=no_owner)) + return stmts + + def drop(self): + """Generate SQL to drop the current sequence + + :return: list of SQL statements + """ + stmts = [] + if self.owner_table is None: + stmts.append(super(Sequence, self).drop()) + return stmts + + +PARTITIONING_STRATEGIES = {'l': 'list', 'r': 'range'} + + +class Table(DbClass): + """A database table definition + + A table is identified by its schema name and table name. It should + have a list of columns. It may have a primary_key, zero or more + foreign_keys, zero or more unique_constraints, and zero or more + indexes. + + A :class:`Table` can also represent a partitioned table or a + partition of a partitioned table. The latter's columns are all + inherited from the parent (partitioned) table, so they are not + shown in an output map (or expected on input). + """ + def __init__(self, name, schema, description, owner, privileges, + tablespace=None, unlogged=False, options=None, + partition_bound_spec=None, partition_by=None, + partition_cols=None, partition_exprs=None, + oid=None): + """Initialize the table + + :param name-privileges: see DbClass.__init__ params + :param tablespace: storage tablespace (from reltablespace) + :param unlogged: unlogged indicator (from relpersistence = 'u') + :param options: access method options (from reloptions) + :param partition_bound_spec: partition bound (from relpartbound) + :param partition_by: partitioning strategy (from partstrat) + """ + super(Table, self).__init__(name, schema, description, owner, + privileges) + self.tablespace = tablespace + self.unlogged = unlogged + self.options = options + if partition_bound_spec is None: + self.partition_bound_spec = None + elif partition_bound_spec.startswith("FOR VALUES "): + self.partition_bound_spec = partition_bound_spec[11:] + else: + self.partition_bound_spec = partition_bound_spec + if partition_by is None: + self.partition_by = None + elif isinstance(partition_by, dict): + partby = list(partition_by.keys())[0] + self.partition_by = partby + self.partition_cols = partition_by[partby] + elif len(partition_by) == 1: + self.partition_by = PARTITIONING_STRATEGIES[partition_by] + else: + assert partition_by in PARTITIONING_STRATEGIES.values() + self.partition_by = partition_by + if partition_cols is not None and len(partition_cols) > 0: + self.partition_cols = [int(n) for n in partition_cols.split(' ')] + elif partition_cols is None and partition_by is None: + self.partition_cols = partition_cols + self.partition_exprs = partition_exprs + self.columns = [] + self.inherits = [] + self.check_constraints = {} + self.primary_key = None + self.foreign_keys = {} + self.unique_constraints = {} + self.indexes = {} + self.triggers = {} + self.rules = {} + self.oid = oid + self._referred_by = [] + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, reloptions AS options, + spcname AS tablespace, relpersistence = 'u' AS unlogged, + rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_expr(relpartbound, c.oid) AS partition_bound_spec, + partstrat AS partition_by, partattrs AS partition_cols, + pg_get_expr(partexprs, pt.partrelid) AS partition_exprs, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + LEFT JOIN pg_tablespace t ON (reltablespace = t.oid) + LEFT JOIN pg_partitioned_table pt ON c.oid = pt.partrelid + WHERE relkind IN ('r', 'p') AND relpersistence != 't' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + AND c.oid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_class'::regclass) + ORDER BY nspname, relname""" + + @staticmethod + def inhquery(): + return """SELECT inhrelid::regclass AS sub, + inhparent::regclass AS parent, inhseqno + FROM pg_inherits + ORDER BY 1, 3""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a table instance from a YAML map + + :param name: table name + :param name: schema map + :param inobj: YAML map of the table + :return: table instance + """ + obj = Table( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('tablespace', None), inobj.pop('unlogged', False), + inobj.pop('options', None), + inobj.pop('partition_bound_spec', None), + inobj.pop('partition_by', None)) + if obj.partition_bound_spec is not None: + obj.inherits = [inobj.get('partition_of')] + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def allprivs(self): + return 'arwdDxt' + + def _normalize_partcols(self): + "Replace integer column numbers by column names" + if isinstance(self.partition_cols[0], int): + self.partition_cols = [self.columns[k - 1].name + for k in self.partition_cols] + + def column_names(self): + """Return a list of column names in the table + + :return: list + """ + return [c.name for c in self.columns] + + def to_map(self, db, dbschemas, opts): + """Convert a table to a YAML-suitable format + + :param dbschemas: database dictionary of schemas + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables or len(self.columns) == 0: + return None + + dct = super(Table, self).to_map(db, opts.no_owner, opts.no_privs, + deepcopy=False) + + for attr in ('tablespace', 'options', 'partition_bound_spec'): + if dct[attr] is None: + dct.pop(attr) + if self.unlogged is False: + dct.pop('unlogged') + if len(self.inherits) == 0: + dct.pop('inherits') + + if self.partition_bound_spec is None: + cols = [] + for column in self.columns: + col = column.to_map(db, opts.no_privs) + if col: + cols.append(col) + dct['columns'] = cols + else: + dct.pop('columns') + assert len(self.inherits) == 1 + dct.update(partition_of=self.inherits[0]) + dct.pop('inherits') + + if self.partition_by is not None: + assert self.partition_cols is not None + self._normalize_partcols() + dct.update(partition_by={self.partition_by: self.partition_cols}) + else: + dct.pop('partition_by') + dct.pop('partition_cols') + dct.pop('partition_exprs') + + if len(self.check_constraints) > 0: + dct['check_constraints'] = copy.copy(dct['check_constraints']) + for k in list(self.check_constraints.values()): + dct['check_constraints'].update( + self.check_constraints[k.name].to_map( + db, self.column_names())) + else: + dct.pop('check_constraints') + if self.primary_key is not None: + dct['primary_key'] = self.primary_key.to_map( + db, self.column_names()) + else: + dct.pop('primary_key') + if len(self.foreign_keys) > 0: + dct['foreign_keys'] = copy.copy(dct['foreign_keys']) + for k in list(self.foreign_keys.values()): + tbls = dbschemas[k.ref_schema].tables + ktable = self.foreign_keys[k.name] + dct['foreign_keys'].update(ktable.to_map( + db, self.column_names(), + tbls[ktable.ref_table].column_names())) + else: + dct.pop('foreign_keys') + if len(self.unique_constraints) > 0: + dct['unique_constraints'] = copy.copy(dct['unique_constraints']) + for k in list(self.unique_constraints.values()): + dct['unique_constraints'].update( + self.unique_constraints[k.name].to_map( + db, self.column_names())) + else: + dct.pop('unique_constraints') + if len(self.indexes) > 0: + idxs = {} + for idx in self.indexes.values(): + if not getattr(idx, '_for_constraint', None): + idxs.update(idx.to_map(db)) + if idxs: + # we may have only indexes not to dump, e.g. the pkey one + dct['indexes'] = idxs + else: + dct.pop('indexes', None) + else: + dct.pop('indexes') + if len(self.rules) > 0: + dct['rules'] = copy.copy(dct['rules']) + for k in list(self.rules.values()): + dct['rules'].update(self.rules[k.name].to_map(db)) + else: + dct.pop('rules') + if len(self.triggers) > 0: + dct['triggers'] = copy.copy(dct['triggers']) + for k in list(self.triggers.values()): + dct['triggers'].update(self.triggers[k.name].to_map(db)) + else: + dct.pop('triggers') + + return copy.deepcopy(dct) + + def create(self, dbversion=None): + """Return SQL statements to CREATE the table + + :return: SQL statements + """ + stmts = [] + cols = [] + colprivs = [] + for col in self.columns: + if not col.inherited: + cols.append(" " + col.add()[0]) + colprivs.append(col.add_privs()) + unlogged = 'UNLOGGED ' if self.unlogged else '' + + partbyclause = partofclause = inhclause = collist = "" + if self.partition_by is not None: + partbyclause = " PARTITION BY %s (%s)" % ( + self.partition_by.upper(), ", ".join(self.partition_cols)) + elif len(self.inherits) > 0: + inhclause = " INHERITS (%s)" % ", ".join( + self.qualname(self.schema, t) for t in self.inherits) + if self.partition_bound_spec is None: + collist = "(\n%s)" % ",\n".join(cols) + else: + partofclause = "PARTITION OF %s FOR VALUES %s" % ( + self.inherits[0], self.partition_bound_spec) + inhclause = "" + + opts = '' + if self.options is not None: + opts = " WITH (%s)" % ', '.join(self.options) + tblspc = '' + if self.tablespace is not None: + tblspc = " TABLESPACE %s" % self.tablespace + stmts.append("CREATE %sTABLE %s %s%s%s%s%s%s" % ( + unlogged, self.qualname(), collist, partbyclause, partofclause, + inhclause, opts, tblspc)) + if self.owner is not None: + stmts.append(self.alter_owner()) + for priv in self.privileges: + stmts.append(add_grant(self, priv)) + if colprivs: + stmts.append(colprivs) + if self.description is not None: + stmts.append(self.comment()) + for col in self.columns: + if col.description is not None: + stmts.append(col.comment()) + if hasattr(self, '_owned_seqs'): + for dep in self._owned_seqs: + stmts.append(dep.add_owner()) + self.created = True + return stmts + + def drop(self): + """Return a SQL DROP statement for the table + + :return: SQL statement + """ + stmts = [] + if not hasattr(self, 'dropped') or not self.dropped: + if hasattr(self, '_dependent_funcs'): + for fnc in self._dependent_funcs: + stmts.append(fnc.drop()) + self.dropped = True + stmts.append("DROP TABLE %s" % self.identifier()) + return stmts + + def diff_options(self, newopts): + """Compare options lists and generate SQL SET or RESET clause + + :newopts: list of new options + :return: SQL SET / RESET clauses + + Generate ([SET|RESET storage_parameter=value) clauses from two + lists in the form of 'key=value' strings. + """ + def to_dict(optlist): + return dict(opt.split('=', 1) for opt in optlist) + + oldopts = {} + if self.options is not None: + oldopts = to_dict(self.options) + newopts = to_dict(newopts) + setclauses = [] + for key, val in list(newopts.items()): + if key not in oldopts: + setclauses.append("%s=%s" % (key, val)) + elif val != oldopts[key]: + setclauses.append("%s=%s" % (key, val)) + resetclauses = [] + for key, val in list(oldopts.items()): + if key not in newopts: + resetclauses.append("%s" % key) + clauses = '' + if setclauses: + clauses = "SET (%s)" % ', '.join(setclauses) + if resetclauses: + clauses += ', ' + if resetclauses: + clauses += "RESET (%s)" % ', '.join(resetclauses) + return clauses + + def alter(self, intable): + """Generate SQL to transform an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + + Compares the table to an input table and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + if len(intable.columns) == 0: + raise KeyError("Table '%s' has no columns" % intable.name) + colnames = [col.name for col in self.columns if not col.dropped] + dbcols = len(colnames) + + colprivs = [] + base = "ALTER %s %s\n " % (self.objtype, self.qualname()) + # check input columns + for (num, incol) in enumerate(intable.columns): + if hasattr(incol, 'oldname'): + assert(self.columns[num].name == incol.oldname) + stmts.append(self.columns[num].rename(incol.name)) + # check existing columns + if num < dbcols and self.columns[num].name == incol.name: + (stmt, descr) = self.columns[num].alter(incol) + if stmt: + stmts.append(base + stmt) + colprivs.append(self.columns[num].diff_privileges(incol)) + if descr: + stmts.append(descr) + # add new columns + elif incol.name not in colnames and not incol.inherited: + (stmt, descr) = incol.add() + stmts.append(base + "ADD COLUMN %s" % stmt) + colprivs.append(incol.add_privs()) + if descr: + stmts.append(descr) + + newopts = [] + if intable.options is not None: + newopts = intable.options + diff_opts = self.diff_options(newopts) + if diff_opts: + stmts.append("ALTER %s %s %s" % (self.objtype, self.identifier(), + diff_opts)) + if colprivs: + stmts.append(colprivs) + if intable.tablespace is not None: + if self.tablespace is None \ + or self.tablespace != intable.tablespace: + stmts.append(base + "SET TABLESPACE %s" + % quote_id(intable.tablespace)) + elif self.tablespace is not None: + stmts.append(base + "SET TABLESPACE pg_default") + + stmts.append(super(Table, self).alter(intable)) + + return stmts + + def alter_drop_columns(self, intable): + """Generate SQL to drop columns from an existing table + + :param intable: a YAML map defining the new table + :return: list of SQL statements + + Compares the table to an input table and generates SQL + statements to drop any columns missing from the one + represented by the input. + """ + if len(intable.columns) == 0: + raise KeyError("Table '%s' has no columns" % intable.name) + stmts = [] + incolnames = set(attr.name for attr in intable.columns) + for attr in self.columns: + if attr.name not in incolnames: + if not getattr(attr, 'inherited', False): + stmts.append(attr.drop()) + + return stmts + + def data_export(self, dbconn, dirpath): + """Copy table data out to a file + + :param dbconn: database connection to use + :param dirpath: full path to the directory for the file to be created + """ + filepath = os.path.join(dirpath, self.extern_filename('data')) + if self.primary_key is not None: + order_by = [quote_id(self.columns[col - 1].name) + for col in self.primary_key.columns] + else: + order_by = ['%d' % (n + 1) for n in range(len(self.columns))] + dbconn.sql_copy_to( + "COPY (SELECT * FROM %s ORDER BY %s) TO STDOUT WITH CSV" % ( + self.qualname(), ', '.join(order_by)), filepath) + + def data_import(self, dirpath): + """Generate SQL to import data into a table + + :param dirpath: full path for the directory for the file + :return: list of SQL statements + """ + filepath = os.path.join(dirpath, self.extern_filename('data')) + stmts = [] + if hasattr(self, '_referred_by'): + for constr in self._referred_by: + stmts.append( + "ALTER TABLE %s DROP CONSTRAINT %s" + % (constr._table.qualname(), constr.name) + ) + stmts.append("TRUNCATE ONLY %s" % self.qualname()) + stmts.append(("\\copy ", self.qualname(), " from '", filepath, + "' csv")) + if hasattr(self, '_referred_by'): + for constr in self._referred_by: + stmts.append(constr.add()) + return stmts + + def get_implied_deps(self, db): + deps = super(Table, self).get_implied_deps(db) + for col in self.columns: + type = db.find_type(col.type) + if type is not None: + deps.add(type) + + # Check if the column depends on a sequence to avoid stating the + # dependency explicitly. + if col.default is not None: + m = re.match(r"nextval\('(.*)'::regclass\)", col.default) + if m: + seq = db.tables.find(m.group(1), self.schema) + if seq: + deps.add(seq) + if seq.owner_table is not None: + if not hasattr(self, '_owned_seqs'): + self._owned_seqs = [] + self._owned_seqs.append(seq) + + for pname in getattr(self, 'inherits', ()): + parent = db.tables.find(pname, self.schema) + assert parent is not None, "couldn't find parent table %s" % pname + deps.add(parent) + + return deps + + +OBJTYPES = ['table', 'sequence', 'view', 'materialized view'] + + +class ClassDict(DbObjectDict): + "The collection of tables and similar objects in a database" + + cls = DbClass + + def _from_catalog(self): + """Initialize the dictionary of tables by querying the catalogs""" + self.cls = Table + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + inhtbls = self.dbconn.fetchall(Table.inhquery()) + self.dbconn.rollback() + for tdata in inhtbls: + tbl = tdata["sub"] + partbl = tdata["parent"] + num = tdata["inhseqno"] + (sch, tbl) = split_schema_obj(tbl) + table = self[(sch, tbl)] + (sch, tbl) = split_schema_obj(partbl) + if table.schema == sch: + partbl = tbl + table.inherits.append(partbl) + self.cls = Sequence + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + obj.get_attrs(self.dbconn) + obj.get_dependent_table(self.dbconn) + from .view import View, MaterializedView + self.cls = View + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + self.cls = MaterializedView + for obj in self.fetch(): + self[obj.key()] = obj + self.by_oid[obj.oid] = obj + + def from_map(self, schema, inobjs, newdb): + """Initialize the dictionary of tables by converting the input map + + :param schema: schema owning the tables + :param inobjs: YAML map defining the schema objects + :param newdb: collection of dictionaries defining the database + """ + from .view import View, MaterializedView + for k in inobjs: + inobj = inobjs[k] + objtype = None + for typ in OBJTYPES: + if k.startswith(typ): + objtype = typ + key = k[len(typ) + 1:] + if objtype is None: + raise KeyError("Unrecognized object type: %s" % k) + if objtype == 'table': + self[(schema.name, key)] = table = Table.from_map( + key, schema, inobj) + if inobj and 'inherits' in inobj: + table.inherits = inobj.pop('inherits') + try: + newdb.columns.from_map(table, inobj['columns']) + except KeyError as exc: + if table.partition_by is not None: + exc.args = ("Table '%s' has no columns" % key, ) + raise + newdb.constraints.from_map(table, inobj) + if 'indexes' in inobj: + newdb.indexes.from_map(table, inobj['indexes']) + if 'rules' in inobj: + newdb.rules.from_map(table, inobj['rules']) + if 'triggers' in inobj: + newdb.triggers.from_map(table, inobj['triggers']) + elif objtype == 'sequence': + self[(schema.name, key)] = Sequence.from_map( + key, schema, inobj) + elif objtype == 'view': + self[(schema.name, key)] = view = View.from_map( + key, schema, inobj) + if 'triggers' in inobj: + newdb.triggers.from_map(view, inobj['triggers']) + elif objtype == 'materialized view': + self[(schema.name, key)] = mview = MaterializedView.from_map( + key, schema, inobj) + if 'indexes' in inobj: + newdb.indexes.from_map(mview, inobj['indexes']) + else: + raise KeyError("Unrecognized object type: %s" % k) + obj = self[(schema.name, key)] + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + + def find(self, obj, schema=None): + """Find a table given its name. + + The name can contain array type modifiers such as '[]' + + Return None if not found. + """ + sch, name = split_schema_obj(obj, schema) + name = name.rstrip('[]') + return self.get((sch, name)) + + def link_refs(self, dbcolumns, dbconstrs, dbindexes, dbrules, dbtriggers): + """Connect columns, constraints, etc. to their respective tables + + :param dbcolumns: dictionary of columns + :param dbconstrs: dictionary of constraints + :param dbindexes: dictionary of indexes + :param dbrules: dictionary of rules + :param dbtriggers: dictionary of triggers + + Links each list of table columns in `dbcolumns` to the + corresponding table. Fills the `foreign_keys`, + `unique_constraints`, `indexes` and `triggers` dictionaries + for each table by traversing the `dbconstrs`, `dbindexes` and + `dbtriggers` dictionaries, which are keyed by schema, table + and constraint, index or trigger name. + """ + seqs = [self[t] for t in self if isinstance(self[t], Sequence)] + for (sch, tbl) in dbcolumns: + if (sch, tbl) in self: + #assert isinstance(self[(sch, tbl)], Table) + self[(sch, tbl)].columns = dbcolumns[(sch, tbl)] + for col in dbcolumns[(sch, tbl)]: + col._table = self[(sch, tbl)] + if col.identity is not None: + for seq in seqs: + if col.table == seq.owner_table and \ + col.name == seq.owner_column: + col._owner_seq = seq + seq._owner_col = col + + # Normalize owner_column's to column names + for (sch, tbl) in self: + table = self[(sch, tbl)] + if isinstance(table, Sequence) and table.owner_table is not None: + if isinstance(table.owner_column, int): + table.owner_column = self[(sch, table.owner_table)]. \ + column_names()[table.owner_column - 1] + + for (sch, tbl, cns) in dbconstrs: + constr = dbconstrs[(sch, tbl, cns)] + if isinstance(constr, CheckConstraint) and constr.is_domain_check: + continue + assert self[(sch, tbl)] + constr._table = table = self[(sch, tbl)] + if isinstance(constr, CheckConstraint): + table.check_constraints.update({cns: constr}) + elif isinstance(constr, PrimaryKey): + table.primary_key = constr + elif isinstance(constr, ForeignKey): + # link referenced and referrer + constr._references = self[( + constr.ref_schema, constr.ref_table)] + self[ + (constr.ref_schema, constr.ref_table) + ]._referred_by.append(constr) + table.foreign_keys.update({cns: constr}) + elif isinstance(constr, UniqueConstraint): + table.unique_constraints.update({cns: constr}) + + def link_one(targdict, schema, tbl, objkey, objtype): + table = self[(schema, tbl)] + if not hasattr(table, objtype): + setattr(table, objtype, {}) + objdict = getattr(table, objtype) + objdict.update({objkey: targdict[(schema, tbl, objkey)]}) + + for (sch, tbl, idx) in dbindexes: + link_one(dbindexes, sch, tbl, idx, 'indexes') + for (sch, tbl, rul) in dbrules: + link_one(dbrules, sch, tbl, rul, 'rules') + dbrules[(sch, tbl, rul)]._table = self[(sch, tbl)] + for (sch, tbl, trg) in dbtriggers: + link_one(dbtriggers, sch, tbl, trg, 'triggers') + dbtriggers[(sch, tbl, trg)]._table = self[(sch, tbl)] diff --git a/pyrseas/dbobject/textsearch.py b/pyrseas/dbobject/textsearch.py new file mode 100644 index 0000000..0576b3a --- /dev/null +++ b/pyrseas/dbobject/textsearch.py @@ -0,0 +1,390 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.textsearch + ~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + This defines eight classes: TSConfiguration, TSDictionary, + TSParser and TSTemplate derived from DbSchemaObject, and + TSConfigurationDict, TSDictionaryDict, TSParserDict and + TSTemplateDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import commentable, ownable, split_schema_obj + + +class TSConfiguration(DbSchemaObject): + """A text search configuration definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_config' + + def __init__(self, name, schema, description, owner, parser, + oid=None): + """Initialize the configuration + + :param name: configuration name (from cfgname) + :param description: comment text (from obj_description()) + :param schema: schema name (from cfgnamespace) + :param owner: owner name (from rolname via cfgowner) + :param parser: parser name (from prsname via cfgparser) + """ + super(TSConfiguration, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.parser = parser + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nc.nspname AS schema, cfgname AS name, + rolname AS owner, np.nspname || '.' || prsname AS parser, + obj_description(c.oid, 'pg_ts_config') AS description, c.oid + FROM pg_ts_config c JOIN pg_roles r ON (r.oid = cfgowner) + JOIN pg_ts_parser p ON (cfgparser = p.oid) + JOIN pg_namespace nc ON (cfgnamespace = nc.oid) + JOIN pg_namespace np ON (prsnamespace = np.oid) + WHERE nc.nspname != 'pg_catalog' + AND nc.nspname != 'information_schema' + ORDER BY nc.nspname, cfgname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a configuration instance from a YAML map + + :param name: configuration name + :param name: schema map + :param inobj: YAML map of the configuration + :return: configuration instance + """ + obj = TSConfiguration( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('parser', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH CONFIGURATION" + + def to_map(self, db, no_owner): + """Convert a text search configuration to a YAML-suitable format + + :return: dictionary + """ + dct = super(TSConfiguration, self).to_map(db, no_owner) + if '.' in self.parser: + (sch, pars) = self.parser.split('.') + if sch == self.schema: + dct['parser'] = pars + return dct + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the configuration + + :return: SQL statements + """ + clauses = [] + clauses.append("PARSER = %s" % self.parser) + return ["CREATE TEXT SEARCH CONFIGURATION %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + def get_implied_deps(self, db): + deps = super(TSConfiguration, self).get_implied_deps(db) + deps.add(db.tsparsers[split_schema_obj(self.parser, self.schema)]) + return deps + + +class TSConfigurationDict(DbObjectDict): + "The collection of text search configurations in a database" + + cls = TSConfiguration + + def from_map(self, schema, inconfigs): + """Initialize the dictionary of configs by examining the input map + + :param schema: schema owning the configurations + :param inconfigs: input YAML map defining the configurations + """ + for key in inconfigs: + if not key.startswith('text search configuration '): + raise KeyError("Unrecognized object type: %s" % key) + tsc = key[26:] + inobj = inconfigs[key] + self[(schema.name, tsc)] = TSConfiguration.from_map( + tsc, schema, inobj) + + +class TSDictionary(DbSchemaObject): + """A text search dictionary definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_dict' + + def __init__(self, name, schema, description, owner, template, options, + oid=None): + """Initialize the dictionary + + :param name: dictionary name (from dictname) + :param schema: schema name (from dictnamespace) + :param description: comment text (from obj_description()) + :param owner: owner name (from rolname via dictowner) + :param template: template name (from dicttemplate) + :param options: initialization option string (from dictinitoption) + """ + super(TSDictionary, self).__init__(name, schema, description) + self._init_own_privs(owner, []) + self.template = template + self.options = options + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, dictname AS name, rolname AS owner, + tmplname AS template, dictinitoption AS options, + obj_description(d.oid, 'pg_ts_dict') AS description, d.oid + FROM pg_ts_dict d JOIN pg_ts_template t ON (dicttemplate = t.oid) + JOIN pg_roles r ON (r.oid = dictowner) + JOIN pg_namespace n ON (dictnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, dictname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a dictionary instance from a YAML map + + :param name: dictionary name + :param name: schema map + :param inobj: YAML map of the dictionary + :return: dictionary instance + """ + obj = TSDictionary( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('template', None), + inobj.pop('options', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH DICTIONARY" + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the dictionary + + :return: SQL statements + """ + clauses = [] + clauses.append("TEMPLATE = %s" % self.template) + if self.options is not None: + clauses.append(self.options) + return ["CREATE TEXT SEARCH DICTIONARY %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSDictionaryDict(DbObjectDict): + "The collection of text search dictionaries in a database" + + cls = TSDictionary + + def from_map(self, schema, indicts): + """Initialize the dictionary of dictionaries by examining the input map + + :param schema: schema owning the dictionaries + :param indicts: input YAML map defining the dictionaries + """ + for key in indicts: + if not key.startswith('text search dictionary '): + raise KeyError("Unrecognized object type: %s" % key) + tsd = key[23:] + inobj = indicts[key] + self[(schema.name, tsd)] = TSDictionary.from_map( + tsd, schema, inobj) + + +class TSParser(DbSchemaObject): + """A text search parser definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_parser' + + def __init__(self, name, schema, description, start, gettoken, end, + headline, lextypes, + oid=None): + """Initialize the parser + + :param name: parser name (from prsname) + :param schema: schema name (from prsnamespace) + :param description: comment text (from obj_description()) + :param start: startup function (from prsstart) + :param gettoken: next-token function (from prstoken) + :param end: shutdown function (from prsend) + :param headline: headline function (from prsheadline) + :param lextypes: lextype function (from prslextype) + """ + super(TSParser, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.start = start + self.gettoken = gettoken + self.end = end + self.headline = headline + self.lextypes = lextypes + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, prsname AS name, + prsstart::regproc AS start, prstoken::regproc AS gettoken, + prsend::regproc AS end, prslextype::regproc AS lextypes, + prsheadline::regproc AS headline, + obj_description(p.oid, 'pg_ts_parser') AS description, p.oid + FROM pg_ts_parser p JOIN pg_namespace n ON (prsnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, prsname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a parser instance from a YAML map + + :param name: parser name + :param name: schema map + :param inobj: YAML map of the parser + :return: parser instance + """ + obj = TSParser(name, schema.name, inobj.pop('description', None), + inobj.pop('start', None), inobj.pop('gettoken', None), + inobj.pop('end', None), inobj.pop('headline', None), + inobj.pop('lextypes', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH PARSER" + + @commentable + @ownable + def create(self, dbversion=None): + """Return SQL statements to CREATE the parser + + :return: SQL statements + """ + clauses = [] + for attr in ['start', 'gettoken', 'end', 'lextypes']: + clauses.append("%s = %s" % (attr.upper(), getattr(self, attr))) + if self.headline is not None: + clauses.append("HEADLINE = %s" % self.headline) + return ["CREATE TEXT SEARCH PARSER %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSParserDict(DbObjectDict): + "The collection of text search parsers in a database" + + cls = TSParser + + def from_map(self, schema, inparsers): + """Initialize the dictionary of parsers by examining the input map + + :param schema: schema owning the parsers + :param inparsers: input YAML map defining the parsers + """ + for key in inparsers: + if not key.startswith('text search parser '): + raise KeyError("Unrecognized object type: %s" % key) + tsp = key[19:] + inobj = inparsers[key] + self[(schema.name, tsp)] = TSParser.from_map(tsp, schema, inobj) + + +class TSTemplate(DbSchemaObject): + """A text search template definition""" + + keylist = ['schema', 'name'] + single_extern_file = True + catalog = 'pg_ts_template' + + def __init__(self, name, schema, description, init, lexize, + oid=None): + """Initialize the template + + :param name: template name (from tmplname) + :param schema: schema name (from tmplnamespace) + :param description: comment text (from obj_description()) + :param init: initialization function (from tmplinit) + :param lexize: lexize function (from tmpllexize) + """ + super(TSTemplate, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.init = init + self.lexize = lexize + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, tmplname AS name, p.oid, + tmplinit::regproc AS init, tmpllexize::regproc AS lexize, + obj_description(p.oid, 'pg_ts_template') AS description + FROM pg_ts_template p + JOIN pg_namespace n ON (tmplnamespace = n.oid) + WHERE nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, tmplname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a template instance from a YAML map + + :param name: template name + :param name: schema map + :param inobj: YAML map of the template + :return: template instance + """ + obj = TSTemplate(name, schema.name, inobj.pop('description', None), + inobj.pop('init', None), inobj.pop('lexize', None)) + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "TEXT SEARCH TEMPLATE" + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the template + + :return: SQL statements + """ + clauses = [] + if self.init is not None: + clauses.append("INIT = %s" % self.init) + clauses.append("LEXIZE = %s" % self.lexize) + return ["CREATE TEXT SEARCH TEMPLATE %s (\n %s)" % ( + self.qualname(), ',\n '.join(clauses))] + + +class TSTemplateDict(DbObjectDict): + "The collection of text search templates in a database" + + cls = TSTemplate + + def from_map(self, schema, intemplates): + """Initialize the dictionary of templates by examining the input map + + :param schema: schema owning the templates + :param intemplates: input YAML map defining the templates + """ + for key in intemplates: + if not key.startswith('text search template '): + raise KeyError("Unrecognized object type: %s" % key) + tst = key[21:] + inobj = intemplates[key] + self[(schema.name, tst)] = TSTemplate.from_map(tst, schema, inobj) diff --git a/pyrseas/dbobject/trigger.py b/pyrseas/dbobject/trigger.py new file mode 100644 index 0000000..0a28557 --- /dev/null +++ b/pyrseas/dbobject/trigger.py @@ -0,0 +1,282 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.trigger + ~~~~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: Trigger derived from + DbSchemaObject, and TriggerDict derived from DbObjectDict. +""" +from . import DbObjectDict, DbSchemaObject +from . import quote_id, commentable, split_schema_obj +from .function import split_schema_func, join_schema_func + +EVENT_TYPES = ['insert', 'delete', 'update', 'truncate'] + + +class Trigger(DbSchemaObject): + """A procedural language trigger""" + + keylist = ['schema', 'table', 'name'] + catalog = 'pg_trigger' + + def __init__(self, name, schema, table, description, procedure, timing, + level, events, constraint=False, deferrable=False, + initially_deferred=False, referencing_new=None, + referencing_old=None, columns=[], condition=None, + arguments='', + oid=None): + """Initialize the trigger + + :param name: trigger name (from tgname) + :param schema: schema name (from tgnamespace) + :param table: table name (from relname via tgrelid) + :param description: comment text (from obj_description()) + :param procedure: function to call (from tgfoid) + :param level: row/statement (from tgtype bit 0) + :param timing: before/after/instead of (from tgtype bit 1 and 6) + :param events: insert/update/delete/truncate (from tgtype bits 2-5) + :param constraint: is it a constraint trigger? (from contype) + :param deferrable: is it deferrable? (from tgdeferrrable) + :param initially_deferred: initially deferred? (from tginitdeferred) + :param referencing_new: NEW transition relation name + :param referencing_old: OLD transition relation name + :param columns: array of column numbers (from tgattr) + :param condition: WHEN condition + :param arguments: arguments to pass to trigger (from tgargs) + """ + super(Trigger, self).__init__(name, schema, description) + self._init_own_privs(None, []) + self.table = table + if procedure[-2:] == '()': + procedure = procedure[:-2] + if '.' in procedure: + self.procedure = split_schema_obj(procedure, self.schema) + else: + self.procedure = procedure + if arguments and '\\000' in arguments: + self.arguments = ", ".join(["'%s'" % a for a in + arguments.split('\\000')[:-1]]) + else: + self.arguments = arguments if len(arguments) > 0 else None + # see Postgres include/catalog/pg_trigger.h + if isinstance(timing, int): + if timing == (1 << 1): + self.timing = 'before' + elif timing == (1 << 6): + self.timing = 'instead of' + else: + self.timing = 'after' + else: + self.timing = timing + self.level = level + if isinstance(events, int): + self.events = [] + for (n, ev) in enumerate(EVENT_TYPES): + if events & 1 << n: + self.events.append(ev) + else: + assert isinstance(events, list), "Events must be a list" + self.events = events + self.constraint = constraint + self.deferrable = deferrable + self.initially_deferred = initially_deferred + self.referencing_new = referencing_new + self.referencing_old = referencing_old + self.columns = columns + self.condition = condition + if condition is not None and condition.startswith('CREATE '): + if 'WHEN (' in condition: + self.condition = condition[condition.index("WHEN (")+6: + condition.index(") EXECUTE ")] + else: + self.condition = None + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS table, tgname AS name, + tgfoid::regprocedure AS procedure, + CASE WHEN tgtype::integer::bit = '1' THEN 'row' + ELSE 'statement' END AS level, + (tgtype::integer::bit(7) & B'1000010')::integer AS timing, + (tgtype >> 2)::integer::bit(4)::integer AS events, + CASE WHEN contype = 't' THEN true ELSE false END AS + constraint, + tgdeferrable AS deferrable, + tginitdeferred AS initially_deferred, tgattr AS columns, + encode(tgargs, 'escape') AS arguments, + pg_get_triggerdef(t.oid) AS condition, + obj_description(t.oid, 'pg_trigger') AS description, + tgnewtable AS referencing_new, + tgoldtable AS referencing_old, + t.oid + FROM pg_trigger t JOIN pg_class c ON (t.tgrelid = c.oid) + JOIN pg_namespace n ON (c.relnamespace = n.oid) + JOIN pg_roles ON (n.nspowner = pg_roles.oid) + LEFT JOIN pg_constraint cn ON (tgconstraint = cn.oid) + WHERE NOT tgisinternal + AND (nspname != 'pg_catalog' AND nspname != 'information_schema') + AND t.tgfoid NOT IN ( + SELECT objid FROM pg_depend WHERE deptype = 'e' + AND classid = 'pg_proc'::regclass) + ORDER BY schema, "table", name""" + + @staticmethod + def from_map(name, table, inobj): + """Initialize a trigger instance from a YAML map + + :param name: trigger name + :param table: table map + :param inobj: YAML map of the trigger + :return: trigger instance + """ + proc = inobj.pop("procedure") + args = "" + if isinstance(proc, str): + if proc[-2:] == "()": + proc = proc[:-2] + elif proc[-1:] == ')': + proc, args = proc[:-1].split('(') + else: # should be a dict + args = proc.pop("arguments", None) + proc = proc.pop("name") + obj = Trigger( + name, table.schema, table.name, inobj.pop('description', None), + proc, inobj.pop('timing', None), inobj.pop('level', 'statement'), + inobj.pop('events', []), inobj.pop('constraint', False), + inobj.pop('deferrable', False), + inobj.pop('initially_deferred', False), + inobj.pop('referencing_new', None), + inobj.pop('referencing_old', None), + inobj.pop('columns', []), inobj.pop('condition', None), args) + obj.set_oldname(inobj) + return obj + + def identifier(self): + """Returns a full identifier for the trigger + + :return: string + """ + return "%s ON %s" % (quote_id(self.name), self._table.qualname()) + + def to_map(self, db): + """Convert a trigger to a YAML-suitable format + + :return: dictionary + """ + dct = super(Trigger, self).to_map(db) + schfunc = join_schema_func(self.procedure) + if self.arguments is not None: + dct["procedure"] = {"name": schfunc, "arguments": self.arguments} + else: + dct["procedure"] = schfunc + dct.pop("arguments") + for attr in ['constraint', 'deferrable', 'initially_deferred']: + if dct[attr] is False: + del dct[attr] + if len(self.columns) > 0: + dct['columns'] = [self._table.column_names()[int(k) - 1] + for k in self.columns.split()] + else: + del dct['columns'] + if self.condition is None: + del dct['condition'] + if self.referencing_new is None: + del dct['referencing_new'] + if self.referencing_old is None: + del dct['referencing_old'] + return {self.name: dct} + + @commentable + def create(self, dbversion=None): + """Return SQL statements to CREATE the trigger + + :return: SQL statements + """ + constr = defer = '' + if self.constraint: + constr = "CONSTRAINT " + if self.deferrable: + defer = "DEFERRABLE " + if self.initially_deferred: + defer += "INITIALLY DEFERRED" + if defer: + defer = '\n ' + defer + evts = " OR ".join(self.events).upper() + if len(self.columns) > 0 and 'update' in self.events: + evts = evts.replace("UPDATE", "UPDATE OF %s" % ( + ", ".join(self.columns))) + cond = '' + if self.condition is not None: + cond = "\n WHEN (%s)" % self.condition + if isinstance(self.procedure, tuple): + procname = "%s.%s" % self.procedure + else: + procname = self.procedure + if self.arguments is None: + args = "" + else: + args = self.arguments + referencing = '' + if self.referencing_new or self.referencing_old: + referencing = "\n REFERENCING" + if self.referencing_new: + referencing += " NEW TABLE AS %s" % self.referencing_new + if self.referencing_old: + referencing += " OLD TABLE AS %s" % self.referencing_old + return ["CREATE %sTRIGGER %s\n %s %s ON %s%s%s\n FOR EACH %s" + "%s\n EXECUTE PROCEDURE %s(%s)" % ( + constr, quote_id(self.name), self.timing.upper(), evts, + self._table.qualname(), defer, referencing, + self.level.upper(), cond, procname, args)] + + def alter(self, inobj): + """Generate SQL to transform an existing trigger + + :param inobj: a YAML map defining the new trigger + :return: list of SQL statements + """ + stmts = [] + if self.procedure != inobj.procedure or \ + self.arguments != inobj.arguments or self.events != inobj.events \ + or self.level != inobj.level or self.timing != inobj.timing \ + or self.referencing_new != inobj.referencing_new \ + or self.referencing_old != inobj.referencing_old: + stmts.append(self.drop()) + stmts.append(inobj.create()) + stmts.append(self.diff_description(inobj)) + return stmts + + def get_implied_deps(self, db): + deps = super(Trigger, self).get_implied_deps(db) + + deps.add(db.tables[self.schema, self.table]) + + # short-circuit augment triggers + if hasattr(self, '_iscfg'): + return deps + + # the trigger procedure can have arguments, but the trigger definition + # has always none (they are accessed through `tg_argv`). + if isinstance(self.procedure, tuple): + fschema, fname = self.procedure + deps.add(db.functions[fschema, fname, '']) + + return deps + + +class TriggerDict(DbObjectDict): + "The collection of triggers in a database" + cls = Trigger + + def from_map(self, table, intriggers): + """Initialize the dictionary of triggers by converting the input map + + :param table: table owning the triggers + :param intriggers: YAML map defining the triggers + """ + for trg in intriggers: + inobj = intriggers[trg] + self[(table.schema, table.name, trg)] = Trigger.from_map( + trg, table, inobj) diff --git a/pyrseas/dbobject/view.py b/pyrseas/dbobject/view.py new file mode 100644 index 0000000..2160601 --- /dev/null +++ b/pyrseas/dbobject/view.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" + pyrseas.dbobject.view + ~~~~~~~~~~~~~~~~~~~~~ + + This module defines two classes: View derived from DbClass and + MaterializedView derived from View. +""" +from pyrseas.yamlutil import MultiLineStr +from . import commentable, ownable, grantable +from .table import DbClass +from .column import Column + + +class View(DbClass): + """A database view definition + + A view is identified by its schema name and view name. + """ + def __init__(self, name, schema, description, owner, privileges, + definition, + oid=None): + """Initialize the view + + :param name-privileges: see DbClass.__init__ params + :param definition: prettified definition (from pg_getviewdef) + """ + super(View, self).__init__(name, schema, description, owner, + privileges) + self.definition = MultiLineStr(definition) + self.triggers = {} + self.columns = [] + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_viewdef(c.oid, TRUE) AS definition, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'v' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a view instance from a YAML map + + :param name: view name + :param name: schema map + :param inobj: YAML map of the view + :return: view instance + """ + obj = View( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('definition', None)) + if "columns" in inobj: + obj.columns = [Column(list(col.keys())[0], schema.name, name, + i + 1, + list(col.values())[0].get("type", None)) + for i, col in enumerate(inobj.get("columns"))] + if 'depends_on' in inobj: + obj.depends_on.extend(inobj['depends_on']) + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + privobjtype = "TABLE" + + @property + def allprivs(self): + return 'arwdDxt' + + def to_map(self, db, opts): + """Convert a view to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return None + dct = super(View, self).to_map(db, opts.no_owner, opts.no_privs) + dct['columns'] = [col.to_map(db, opts.no_privs) + for col in self.columns] + if 'dependent_funcs' in dct: + dct.pop('dependent_funcs') + if len(self.triggers) > 0: + for key in list(self.triggers.values()): + dct['triggers'].update(self.triggers[key.name].to_map(db)) + else: + dct.pop('triggers') + return dct + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newdefn=None): + """Return SQL statements to CREATE the view + + :return: SQL statements + """ + defn = newdefn or self.definition + if defn[-1:] == ';': + defn = defn[:-1] + return ["CREATE%s VIEW %s AS\n %s" % ( + newdefn and " OR REPLACE" or '', self.qualname(), defn)] + + def alter(self, inview, dbversion=None): + """Generate SQL to transform an existing view + + :param inview: a YAML map defining the new view + :return: list of SQL statements + + Compares the view to an input view and generates SQL + statements to transform it into the one represented by the + input. + """ + stmts = [] + for col in self.columns: + if col.name != inview.columns[col.number - 1].name: + raise KeyError("Cannot change name of view column '%s'" + % col.name) + if col.type != inview.columns[col.number - 1].type: + raise TypeError("Cannot change datatype of view column '%s'" + % col.name) + if self.definition != inview.definition: + stmts.append(self.create(dbversion, inview.definition)) + stmts.append(super(View, self).alter(inview)) + return stmts + + +class MaterializedView(View): + """A materialized view definition + + A materialized view is identified by its schema name and view name. + """ + def __init__(self, name, schema, description, owner, privileges, + definition, with_data=False, + oid=None): + """Initialize the materialized view + + :param name-privileges: see DbClass.__init__ params + :param definition: prettified definition (from pg_getviewdef) + :param with_data: is view populated (from relispopulated) + """ + super(MaterializedView, self).__init__( + name, schema, description, owner, privileges, definition) + self.with_data = with_data + self.indexes = {} + self.oid = oid + + @staticmethod + def query(dbversion=None): + return """ + SELECT nspname AS schema, relname AS name, rolname AS owner, + array_to_string(relacl, ',') AS privileges, + pg_get_viewdef(c.oid, TRUE) AS definition, + relispopulated AS with_data, + obj_description(c.oid, 'pg_class') AS description, c.oid + FROM pg_class c JOIN pg_roles r ON (r.oid = relowner) + JOIN pg_namespace ON (relnamespace = pg_namespace.oid) + WHERE relkind = 'm' + AND nspname != 'pg_catalog' AND nspname != 'information_schema' + ORDER BY nspname, relname""" + + @staticmethod + def from_map(name, schema, inobj): + """Initialize a materialized view instance from a YAML map + + :param name: view name + :param name: schema map + :param inobj: YAML map of the view + :return: materialized view instance + """ + obj = MaterializedView( + name, schema.name, inobj.pop('description', None), + inobj.pop('owner', None), inobj.pop('privileges', []), + inobj.pop('definition', None)) + if "columns" in inobj: + obj.columns = [Column(list(col.keys())[0], schema.name, name, + i + 1, + list(col.values())[0].get("type", None)) + for i, col in enumerate(inobj.get("columns"))] + obj.fix_privileges() + obj.set_oldname(inobj) + return obj + + @property + def objtype(self): + return "MATERIALIZED VIEW" + + def to_map(self, db, opts): + """Convert a materialized view to a YAML-suitable format + + :param opts: options to include/exclude tables, etc. + :return: dictionary + """ + if hasattr(opts, 'excl_tables') and opts.excl_tables \ + and self.name in opts.excl_tables: + return None + mvw = super(MaterializedView, self).to_map(db, opts) + if len(self.indexes) > 0: + for k in list(self.indexes.values()): + mvw['indexes'].update(self.indexes[k.name].to_map(db)) + else: + mvw.pop('indexes') + return mvw + + @commentable + @grantable + @ownable + def create(self, dbversion=None, newdefn=None): + """Return SQL statements to CREATE the materialized view + + :return: SQL statements + """ + defn = newdefn or self.definition + if defn[-1:] == ';': + defn = defn[:-1] + return ["CREATE %s %s AS\n %s" % ( + self.objtype, self.qualname(), defn)] diff --git a/pyrseas/dbtoyaml.py b/pyrseas/dbtoyaml.py new file mode 100644 index 0000000..5f50b32 --- /dev/null +++ b/pyrseas/dbtoyaml.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""dbtoyaml - extract the schema of a PostgreSQL database in YAML format""" + +from __future__ import print_function +import sys + +from pyrseas import __version__ +from pyrseas.yamlutil import yamldump +from pyrseas.database import Database +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(schema=None): + """Convert database table specifications to YAML.""" + parser = cmd_parser("Extract the schema of a PostgreSQL database in " + "YAML format", __version__) + parser.add_argument('-m', '--multiple-files', action='store_true', + help='output to multiple files (metadata directory)') + parser.add_argument('-O', '--no-owner', action='store_true', + help='exclude object ownership information') + parser.add_argument('-x', '--no-privileges', action='store_true', + dest='no_privs', + help='exclude privilege (GRANT/REVOKE) information') + group = parser.add_argument_group("Object inclusion/exclusion options", + "(each can be given multiple times)") + group.add_argument('-n', '--schema', metavar='SCHEMA', dest='schemas', + action='append', default=[], + help="extract the named schema(s) (default all)") + group.add_argument('-N', '--exclude-schema', metavar='SCHEMA', + dest='excl_schemas', action='append', default=[], + help="do NOT extract the named schema(s) " + "(default none)") + group.add_argument('-t', '--table', metavar='TABLE', dest='tables', + action='append', default=[], + help="extract the named table(s) (default all)") + group.add_argument('-T', '--exclude-table', metavar='TABLE', + dest='excl_tables', action='append', default=[], + help="do NOT extract the named table(s) " + "(default none)") + parser.set_defaults(schema=schema) + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + if options.multiple_files and output: + parser.error("Cannot specify both --multiple-files and --output") + + db = Database(cfg) + dbmap = db.to_map() + + if not options.multiple_files: + print(yamldump(dbmap), file=output or sys.stdout) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/lib/__init__.py b/pyrseas/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyrseas/lib/dbconn.py b/pyrseas/lib/dbconn.py new file mode 100644 index 0000000..ce0776c --- /dev/null +++ b/pyrseas/lib/dbconn.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +""" + lib.dbconn + ~~~~~~~~~~ + + A `DbConnection` is a helper class representing a connection to a + PostgreSQL database. +""" +import sys + +from psycopg import connect +from psycopg.extras import dict_row + + +class DbConnection(object): + """A database connection, possibly disconnected""" + + def __init__(self, dbname, user=None, pswd=None, host=None, port=None): + """Initialize the connection information + + :param dbname: database name + :param user: user name + :param pswd: user password + :param host: host name + :param port: host port number + """ + self.dbname = dbname + self.user = '' if user is None else " user=%s" % user + self.pswd = '' if pswd is None else " password=%s" % pswd + self.host = '' if host is None else "host=%s " % host + self.port = '' if port is None else "port=%d " % port + self.conn = None + + def connect(self): + """Connect to the database""" + try: + self.conn = connect("%s%sdbname=%s%s%s" % ( + self.host, self.port, self.dbname, self.user, self.pswd)) + except Exception as exc: + if str(exc)[:6] == 'FATAL:': + sys.exit("Database connection error: %s" % str(exc)[8:]) + else: + raise exc + + def close(self): + """Close the database connection""" + if self.conn and not self.conn.closed: + self.conn.close() + self.conn = None + + def commit(self): + """Commit currently open transaction""" + self.conn.commit() + + def rollback(self): + """Roll back currently open transaction""" + self.conn.rollback() + + def execute(self, query, args=None): + """Create a cursor, execute a query and return the cursor + + :param query: text of the statement to execute + :param args: arguments to query + :return: cursor + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + try: + curs.execute(query, args) + except Exception as exc: + self.conn.rollback() + curs.close() + raise exc + return curs + + def fetchone(self, query, args=None): + """Execute a single row SELECT query and return row + + :param query: a SELECT query to be executed + :param args: arguments to query + :return: a psycopg DictRow + + The cursor is closed. + """ + curs = self.execute(query, args) + row = curs.fetchone() + curs.close() + return row + + def fetchall(self, query, args=None): + """Execute a SELECT query and return rows + + :param query: a SELECT query to be executed + :param args: arguments to query + :return: a list of psycopg DictRow's + + The cursor is closed. + """ + curs = self.execute(query, args) + rows = curs.fetchall() + curs.close() + return rows + + def sql_copy_to(self, sql, path): + """Execute an SQL COPY command to a file + + :param sql: SQL copy command + :param path: file name/path to copy into + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + with curs.copy(sql) as copy: + with open(path, "wb") as f: + for data in copy: + f.write(bytes(data)) + + def copy_from(self, path, table): + """Execute a COPY command from a file in CSV format + + :param path: file name/path to copy from + :param table: possibly schema qualified table name + """ + if self.conn is None or self.conn.closed: + self.connect() + curs = self.conn.cursor() + with open(path, 'r') as f: + try: + with curs.copy("COPY %s FROM STDIN WITH CSV" % table) as copy: + while data := f.read(): + copy.write(data) + except: + raise + curs.close() diff --git a/pyrseas/lib/dbutils.py b/pyrseas/lib/dbutils.py new file mode 100644 index 0000000..e553ce0 --- /dev/null +++ b/pyrseas/lib/dbutils.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +"""Database utility functions and classes + +These are primarily to assist in testing Pyrseas, i.e., without having +to depend on the application-level DbConnection. +""" +import os + +from psycopg import connect +from psycopg.rows import dict_row + + +def pgconnect(dbname, user=None, host=None, port=None, autocommit=False): + "Connect to a Postgres database using psycopg" + user = '' if user is None else " user=%s" % user + host = '' if host is None else "host=%s " % host + port = '' if port is None else "port=%d " % port + return connect("%s%sdbname=%s%s" % (host, port, dbname, user), + autocommit=autocommit) + + +def pgexecute(dbconn, oper, args=None): + "Execute an operation using a cursor" + curs = dbconn.cursor() + try: + curs.execute(oper, args) + except: + curs.close() + dbconn.rollback() + raise + return curs + + +ADMIN_DB = os.environ.get("PG_ADMIN_DB", 'postgres') +CREATE_DDL = "CREATE DATABASE %s TEMPLATE = template0" + + +class PostgresDb(object): + """A PostgreSQL database connection + + This is separate from the one used by DbConnection, because tests + need to create and drop databases and other objects, + independently. + """ + def __init__(self, name, user, host, port): + self.name = name + self.conn = None + self.user = user + self.host = host + self.port = port and int(port) + self._version = 0 + + def connect(self): + """Connect to the database + + If we're not already connected we first connect to the admin + database and see if the given database exists. If it doesn't, + we create and then connect to it. + """ + if not self.conn: + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port) + curs = pgexecute(conn, + "SELECT 1 FROM pg_database WHERE datname = '%s'" % + self.name) + row = curs.fetchone() + if not row: + curs.close() + conn2 = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn2, CREATE_DDL % self.name) + curs.close() + conn.close() + self.conn = pgconnect(self.name, self.user, self.host, self.port) + curs = pgexecute(self.conn, "SHOW server_version_num") + vers = curs.fetchone() + self._version = int(vers["server_version_num"]) + + def close(self): + "Close the connection if still open" + if not self.conn: + return ValueError + self.conn.close() + + @property + def version(self): + return self._version + + def create(self): + "Drop the database if it exists and re-create it" + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn, "DROP DATABASE IF EXISTS %s" % self.name) + curs = pgexecute(conn, CREATE_DDL % self.name) + curs.close() + conn.close() + + def drop(self): + "Drop the database" + conn = pgconnect(ADMIN_DB, self.user, self.host, self.port, + autocommit=True) + curs = pgexecute(conn, "DROP DATABASE %s" % self.name) + curs.close() + conn.close() + + def execute(self, stmt, args=None): + "Execute a DDL statement" + curs = pgexecute(self.conn, stmt, args) + curs.close() + + def execute_commit(self, stmt, args=None): + "Execute a DDL statement and commit" + self.execute(stmt, args) + self.conn.commit() + + def fetchone(self, query, args=None): + "Execute a query and return one row" + try: + curs = pgexecute(self.conn, query, args) + except Exception as exc: + raise exc + row = curs.fetchone() + curs.close() + return row diff --git a/pyrseas/testutils.py b/pyrseas/testutils.py new file mode 100644 index 0000000..bbcee1a --- /dev/null +++ b/pyrseas/testutils.py @@ -0,0 +1,412 @@ +# -*- coding: utf-8 -*- +"""Utility functions and classes for testing Pyrseas""" + +import sys +import os +import getpass +import tempfile +import glob +import subprocess +from unittest import TestCase + +import yaml + +from pyrseas.config import Config +from pyrseas.database import Database +from pyrseas.augmentdb import AugmentDatabase +from pyrseas.lib.dbutils import pgexecute, PostgresDb + + +def fix_indent(stmt): + "Fix specifications which are in a new line with indentation" + return stmt.replace(' ', ' ').replace(' ', ' ').replace('\n ', ' '). \ + replace('( ', '(') + + +def remove_temp_files(tmpdir, prefix=''): + "Remove files in a temporary directory" + for tfile in glob.glob(os.path.join(tmpdir, prefix + '*')): + if os.path.isdir(tfile): + for entry in os.listdir(tfile): + entry = os.path.join(tmpdir, tfile, entry) + if os.path.isdir(entry): + for file in os.listdir(entry): + os.remove(os.path.join(entry, file)) + os.rmdir(entry) + else: + os.remove(entry) + os.rmdir(tfile) + else: + os.remove(tfile) + + +TEST_DBNAME = os.environ.get("PYRSEAS_TEST_DB", 'pyrseas_testdb') +TEST_USER = os.environ.get("PYRSEAS_TEST_USER", getpass.getuser()) +TEST_HOST = os.environ.get("PYRSEAS_TEST_HOST", None) +TEST_PORT = int(os.environ.get("PYRSEAS_TEST_PORT", 5432)) +PG_OWNER = 'postgres' +TEST_DIR = os.path.join(tempfile.gettempdir(), + os.environ.get("PYRSEAS_TEST_DIR", 'pyrseas_test')) +TRAVIS = (os.environ.get("TRAVIS", 'false') == 'true') + + +class PgTestDb(PostgresDb): + """A PostgreSQL database connection for testing.""" + + def clear(self): + "Drop schemas and other objects" + STD_DROP = 'DROP %s IF EXISTS "%s" CASCADE' + # Schemas + curs = pgexecute( + self.conn, + """SELECT nspname FROM pg_namespace + WHERE nspname != 'information_schema' + AND substring(nspname for 3) != 'pg_' + ORDER BY nspname""") + objs = curs.fetchall() + curs.close() + self.conn.rollback() + for obj in objs: + self.execute(STD_DROP % ('SCHEMA', obj["nspname"])) + self.conn.commit() + + # Extensions + curs = pgexecute( + self.conn, + """SELECT extname FROM pg_extension + JOIN pg_namespace n ON (extnamespace = n.oid) + WHERE nspname != 'information_schema' + AND extname != 'plpgsql'""") + exts = curs.fetchall() + curs.close() + self.conn.rollback() + for ext in exts: + self.execute(STD_DROP % ('EXTENSION', ext["extname"])) + self.conn.commit() + + # User mappings + curs = pgexecute( + self.conn, + """SELECT CASE umuser WHEN 0 THEN 'PUBLIC' ELSE + pg_get_userbyid(umuser) END AS username, s.srvname + FROM pg_user_mappings u + JOIN pg_foreign_server s ON (srvid = s.oid)""") + umaps = curs.fetchall() + curs.close() + self.conn.rollback() + for ump in umaps: + self.execute('DROP USER MAPPING IF EXISTS FOR "%s" SERVER "%s"' % ( + ump["username"], ump["srvname"])) + self.conn.commit() + + # Servers + curs = pgexecute(self.conn, "SELECT srvname FROM pg_foreign_server") + servs = curs.fetchall() + curs.close() + self.conn.rollback() + for srv in servs: + self.execute(STD_DROP % ('SERVER', srv["srvname"])) + self.conn.commit() + + # Foreign data wrappers + curs = pgexecute(self.conn, + "SELECT fdwname FROM pg_foreign_data_wrapper") + fdws = curs.fetchall() + curs.close() + self.conn.rollback() + for fdw in fdws: + self.execute(STD_DROP % ('FOREIGN DATA WRAPPER', fdw["fdwname"])) + self.conn.commit() + + # Create default schema + self.execute("CREATE SCHEMA sd") + self.execute("set search_path='sd', 'pg_catalog'") + self.conn.commit() + + def is_plpgsql_installed(self): + "Is PL/pgSQL installed?" + curs = pgexecute(self.conn, + "SELECT 1 FROM pg_language WHERE lanname = 'plpgsql'") + row = curs.fetchone() + curs.close() + return row and True + + def is_superuser(self): + "Is current user a superuser?" + curs = pgexecute(self.conn, "SELECT 1 FROM pg_roles WHERE rolsuper " + "AND rolname = CURRENT_USER ") + row = curs.fetchone() + curs.close() + return row and True + + +def _connect_clear(dbname): + db = PgTestDb(dbname, TEST_USER, TEST_HOST, TEST_PORT) + db.connect() + db.clear() + return db + + +class PyrseasTestCase(TestCase): + """Base class for most test cases""" + + def setUp(self): + self.db = _connect_clear(TEST_DBNAME) + self.cfg = Config(sys_only=True) + if 'database' not in self.cfg: + self.cfg.update(database={}) + dbc = self.cfg['database'] + dbc['dbname'] = self.db.name + dbc['username'] = self.db.user + dbc['password'] = None + dbc['host'] = self.db.host + dbc['port'] = self.db.port + + def tearDown(self): + self.db.close() + + def database(self): + """The Pyrseas Database instance""" + return Database(self.cfg) + + def config_options(self, **kwargs): + class Opts(): + def __init__(self, **kwargs): + [setattr(self, opt, val) for opt, val in list(kwargs.items())] + self.cfg['options'] = Opts(**kwargs) + + +class DatabaseToMapTestCase(PyrseasTestCase): + """Base class for "database to map" test cases""" + + superuser = False + + def to_map(self, stmts, config={}, schemas=[], tables=[], no_owner=True, + no_privs=True, superuser=False, multiple_files=False): + """Execute statements and return a database map. + + :param stmts: list of SQL statements to execute + :param config: dictionary of configuration information + :param schemas: list of schemas to map + :param tables: list of tables to map + :param no_owner: exclude object owner information + :param no_privs: exclude privilege information + :param superuser: must be superuser to run + :param multiple_files: emulate --multiple_files option + :return: possibly trimmed map of database + """ + if (self.superuser or superuser) and not self.db.is_superuser(): + self.skipTest("Must be a superuser to run this test") + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + if multiple_files: + self.cfg.merge({'files': {'metadata_path': os.path.join( + TEST_DIR, self.cfg['repository']['metadata'])}}) + if 'datacopy' in config: + self.cfg.merge({'files': {'data_path': os.path.join( + TEST_DIR, self.cfg['repository']['data'])}}) + self.config_options(schemas=schemas, tables=tables, no_owner=no_owner, + no_privs=no_privs, multiple_files=multiple_files) + self.cfg.merge(config) + return self.database().to_map() + + def yaml_load(self, filename, subdir=None): + """Read a file in the metadata_path and process it with YAML load + + :param filename: name of the file + :param subdir: name of a subdirectory where the file is located + :return: YAML dictionary + """ + with open(os.path.join(self.cfg['files']['metadata_path'], + subdir or '', filename), 'r') as f: + inmap = f.read() + return yaml.safe_load(inmap) + + def remove_tempfiles(self): + remove_temp_files(TEST_DIR) + + @staticmethod + def sort_privileges(data): + try: + sorted_privlist = [] + for sortedItem in sorted([list(i.keys())[0] + for i in data['privileges']]): + sorted_privlist.append( + [item for item in data['privileges'] + if list(item.keys())[0] == sortedItem][0]) + data['privileges'] = sorted_privlist + finally: + return data + + +class InputMapToSqlTestCase(PyrseasTestCase): + """Base class for "input map to SQL" test cases""" + + superuser = False + + def to_sql(self, inmap, stmts=None, config={}, superuser=False, schemas=[], + revert=False, quote_reserved=False): + """Execute statements and compare database to input map. + + :param inmap: dictionary defining target database + :param stmts: list of SQL database setup statements + :param config: dictionary of configuration information + :param superuser: indicates test requires superuser privilege + :param schemas: list of schemas to diff + :param revert: generate statements to back out changes + :param quote_reserved: fetch reserved words + :return: list of SQL statements + """ + if (self.superuser or superuser) and not self.db.is_superuser(): + self.skipTest("Must be a superuser to run this test") + if stmts: + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + + if 'datacopy' in config: + self.cfg.merge({'files': {'data_path': os.path.join( + TEST_DIR, self.cfg['repository']['data'])}}) + self.config_options(schemas=schemas, revert=revert), + self.cfg.merge(config) + return self.database().diff_map(inmap, quote_reserved=quote_reserved) + + def std_map(self, plpgsql_installed=False): + "Return a standard schema map for the default database" + base = {'schema sd': { + 'owner': self.db.user, + 'privileges': []}} + base.update({'extension plpgsql': { + 'schema': 'pg_catalog', 'owner': PG_OWNER, + 'description': "PL/pgSQL procedural language"}}) + return base + + +TEST_DBNAME_SRC = os.environ.get("PYRSEAS_TEST_DB_SRC", 'pyrseas_testdb_src') + + +class DbMigrateTestCase(TestCase): + + @classmethod + def setUpClass(cls): + cls.srcdb = _connect_clear(TEST_DBNAME_SRC) + cls.db = _connect_clear(TEST_DBNAME) + progdir = os.path.abspath(os.path.dirname(__file__)) + cls.dbtoyaml = os.path.join(progdir, 'dbtoyaml.py') + cls.yamltodb = os.path.join(progdir, 'yamltodb.py') + cls.tmpdir = TEST_DIR + if not os.path.exists(cls.tmpdir): + os.mkdir(cls.tmpdir) + + def add_public_schema(self, db): + db.execute("CREATE SCHEMA IF NOT EXISTS public") + db.execute("ALTER SCHEMA public OWNER TO postgres") + db.execute("COMMENT ON SCHEMA public IS " + "'standard public schema'") + db.execute("DROP SCHEMA IF EXISTS sd") + db.conn.commit() + + def remove_public_schema(self, db): + db.execute("DROP SCHEMA IF EXISTS public CASCADE") + db.conn.commit() + + @classmethod + def remove_tempfiles(cls, prefix): + remove_temp_files(cls.tmpdir, prefix) + + def execute_script(self, path, scriptname): + scriptfile = os.path.join(os.path.abspath(os.path.dirname(path)), + scriptname) + lines = [] + with open(scriptfile, 'r') as fd: + lines = [line.strip() for line in fd if line != '\n' and + not line.startswith('--')] + self.srcdb.execute_commit(' '.join(lines)) + + def tempfile_path(self, filename): + return os.path.join(self.tmpdir, filename) + + def _db_params(self): + args = [] + if self.db.host is not None: + args.append("--host=%s" % self.db.host) + if self.db.port is not None: + args.append("--port=%d " % self.db.port) + if self.db.user is not None: + args.append("--username=%s" % self.db.user) + return args + + def lines(self, the_file): + with open(the_file) as f: + lines = f.readlines() + return lines + + def run_pg_dump(self, dumpfile, srcdb=False, incldata=False): + """Run pg_dump using special scripts or directly (on Travis-CI) + + :param dumpfile: path to the pg_dump output file + :param srcdb: run against source database + """ + if TRAVIS: + pg_dumpver = 'pg_dump' + else: + v = self.srcdb._version + pg_dumpver = "pg_dump%d%d" % (v // 10000, + (v - v // 10000 * 10000) // 100) + if sys.platform == 'win32': + pg_dumpver += '.bat' + dbname = self.srcdb.name if srcdb else self.db.name + args = [pg_dumpver] + args.extend(self._db_params()) + if not incldata: + args.extend(['-s']) + args.extend(['-f', dumpfile, dbname]) + subprocess.check_call(args) + + def invoke(self, args): + args.insert(0, sys.executable) + path = [os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))] + path.append(os.path.abspath(os.path.join(os.path.dirname( + yaml.__file__), '..'))) + env = os.environ.copy() + env.update({'PYTHONPATH': os.pathsep.join(path)}) + subprocess.check_call(args, env=env) + + def create_yaml(self, yamlfile='', srcdb=False): + dbname = self.srcdb.name if srcdb else self.db.name + args = [self.dbtoyaml] + args.extend(self._db_params()) + if yamlfile: + args.extend(['-o', yamlfile, dbname]) + else: + args.extend(['-r', TEST_DIR, '-m', dbname]) + self.invoke(args) + + def migrate_target(self, yamlfile, outfile): + args = [self.yamltodb] + args.extend(self._db_params()) + if yamlfile: + args.extend(['-u', '-o', outfile, self.db.name, yamlfile]) + else: + args.extend(['-u', '-o', outfile, '-r', TEST_DIR, '-m', + self.db.name]) + self.invoke(args) + + +class AugmentToMapTestCase(PyrseasTestCase): + + def to_map(self, stmts, augmap): + """Apply an augment map and return a map of the updated database. + + :param stmts: list of SQL statements to execute + :param augmap: dictionary describing the augmentations + :return: dictionary of the updated database + """ + for stmt in stmts: + self.db.execute(stmt) + self.db.conn.commit() + self.config_options(schemas=[], tables=[], no_owner=True, + no_privs=True, multiple_files=False) + db = AugmentDatabase(self.cfg) + return db.apply(augmap) diff --git a/pyrseas/yamltodb.py b/pyrseas/yamltodb.py new file mode 100644 index 0000000..d0d793c --- /dev/null +++ b/pyrseas/yamltodb.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""yamltodb - generate SQL statements to update a PostgreSQL database +to match the schema specified in a YAML file""" + +from __future__ import print_function +import sys +from argparse import FileType + +import yaml + +from pyrseas import __version__ +from pyrseas.database import Database +from pyrseas.cmdargs import cmd_parser, parse_args + + +def main(): + """Convert YAML specifications to database DDL.""" + parser = cmd_parser("Generate SQL statements to update a PostgreSQL " + "database to match the schema specified in a " + "YAML-formatted file(s)", __version__) + parser.add_argument('-m', '--multiple-files', action='store_true', + help='input from multiple files (metadata directory)') + parser.add_argument('spec', nargs='?', type=FileType('r'), + default=sys.stdin, help='YAML specification') + parser.add_argument('-1', '--single-transaction', action='store_true', + dest='onetrans', help="wrap commands in BEGIN/COMMIT") + parser.add_argument('-u', '--update', action='store_true', + help="apply changes to database (implies -1)") + parser.add_argument('--revert', action='store_true', + help="generate SQL to revert changes (experimental)") + parser.add_argument('-n', '--schema', metavar='SCHEMA', dest='schemas', + action='append', default=[], + help="process only named schema(s) (default all)") + cfg = parse_args(parser) + output = cfg['files']['output'] + options = cfg['options'] + db = Database(cfg) + if options.multiple_files: + inmap = db.map_from_dir() + else: + try: + inmap = yaml.safe_load(options.spec) + except Exception as exc: + print("Unable to process the input YAML file") + print("Error is '%s'" % exc) + return 1 + + stmts = db.diff_map(inmap) + if stmts: + fd = output or sys.stdout + if options.onetrans or options.update: + print("BEGIN;", file=fd) + for stmt in stmts: + if isinstance(stmt, tuple): + outstmt = "".join(stmt) + '\n' + else: + outstmt = "%s;\n" % stmt + print(outstmt, file=fd) + if options.onetrans or options.update: + print("COMMIT;", file=fd) + if options.update: + try: + for stmt in stmts: + if isinstance(stmt, tuple): + # expected format: (\copy, table, from, path, csv) + db.dbconn.copy_from(stmt[3], stmt[1]) + else: + db.dbconn.execute(stmt) + except: + db.dbconn.rollback() + raise + else: + db.dbconn.commit() + print("Changes applied", file=sys.stderr) + if output: + output.close() + +if __name__ == '__main__': + main() diff --git a/pyrseas/yamlutil.py b/pyrseas/yamlutil.py new file mode 100644 index 0000000..febed8c --- /dev/null +++ b/pyrseas/yamlutil.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +"""Pyrseas YAML utilities""" + +from yaml import add_representer, dump + +class MultiLineStr(str): + """ Marker for multiline strings""" + + +def MultiLineStr_presenter(dumper, data): + return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|') +add_representer(MultiLineStr, MultiLineStr_presenter) + + +def yamldump(objmap): + """Dump an object map using yaml.dump with certain defaults + + :param objmap: dictionary + :return: dumped object map + """ + return dump(objmap, default_flow_style=False, allow_unicode=True) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2361c86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +PyYAML>=5.3 +psycopg>=3.1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..3c6e79c --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d54b922 --- /dev/null +++ b/setup.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Pyrseas - Utilities to assist with database schema versioning. +""" +import sys + +from setuptools import setup +from setuptools.command.test import test as TestCommand + + +class PyTest(TestCommand): + + def finalize_options(self): + TestCommand.finalize_options(self) + self.test_args = [] + self.test_suite = True + + def run_tests(self): + import pytest + errno = pytest.main(self.test_args) + sys.exit(errno) + + +setup( + name='Pyrseas', + version='0.10.0', + packages=['pyrseas', 'pyrseas.dbobject', 'pyrseas.lib', 'pyrseas.augment', + ], + package_data={'pyrseas': ['config.yaml']}, + entry_points={ + 'console_scripts': [ + 'dbtoyaml = pyrseas.dbtoyaml:main', + 'yamltodb = pyrseas.yamltodb:main', + 'dbaugment = pyrseas.dbaugment:main']}, + + install_requires=[ + 'psycopg >= 3.1', + 'PyYAML >= 5.3'], + + tests_require=['pytest'], + cmdclass={'test': PyTest}, + + author='Joe Abbate', + author_email='jma@freedomcircle.com', + description='Utilities to assist in database schema versioning', + long_description=open('README.rst').read(), + url='https://perseas.github.io/', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: BSD License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: SQL', + 'Topic :: Database :: Front-Ends', + 'Topic :: Software Development :: Code Generators', + 'Topic :: Software Development :: Version Control'], + platforms='OS-independent', + license='BSD') diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..db22637 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +# to avoid setuptools/distutils bug diff --git a/tests/augment/test_audit.py b/tests/augment/test_audit.py new file mode 100644 index 0000000..8107290 --- /dev/null +++ b/tests/augment/test_audit.py @@ -0,0 +1,274 @@ +# -*- coding: utf-8 -*- +"""Test audit columns""" + +import pytest + +from pyrseas.testutils import AugmentToMapTestCase + +CREATE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +FUNC_SRC1 = """ +BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP; + RETURN NEW; +END""" + +FUNC_SRC2 = """ +BEGIN + NEW.updated = CURRENT_TIMESTAMP; + RETURN NEW; +END""" + + +class AuditColumnsTestCase(AugmentToMapTestCase): + """Test mapping of audit column augmentations""" + + def test_predef_column(self): + "Add predefined audit column" + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'created_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'created_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_unknown_table(self): + "Error on non-existent table" + augmap = {'schema sd': {'table t2': { + 'audit_columns': 'created_date_only'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_bad_audit_spec(self): + "Error on bad audit column specification" + augmap = {'schema sd': {'table t1': { + 'audit_column': 'created_date_only'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_unknown_audit_spec(self): + "Error on non-existent audit column specification" + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'created_date'}}} + with pytest.raises(KeyError): + self.to_map([CREATE_STMT], augmap) + + def test_new_column(self): + "Add new (non-predefined) audit column" + augmap = {'augmenter': {'columns': { + 'modified_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}, + 'audit_columns': {'modified_date_only': { + 'columns': ['modified_date']}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'modified_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_date': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_rename_column(self): + "Add predefined audit column but with new name" + augmap = {'augmenter': {'columns': { + 'modified_timestamp': {'name': 'updated'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'modified_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + colmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'updated': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_modified_only': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.audit_modified', 'timing': 'before'}}} + funcmap = {'language': 'plpgsql', 'returns': 'trigger', + 'security_definer': True, 'description': + 'Provides modified_timestamp values for audit columns.', + 'source': FUNC_SRC2} + assert dbmap['schema sd']['table t1'] == colmap + assert dbmap['schema sd']['function audit_modified()'] == funcmap + + def test_change_column_type(self): + "Add predefined audit column but with changed datatype" + augmap = {'augmenter': {'columns': {'created_date': {'type': 'text'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'created_date_only'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'created_date': {'type': 'text', 'not_null': True, + 'default': "('now'::text)::date"}}]} + assert expmap == dbmap['schema sd']['table t1'] + + def test_columns_with_trigger(self): + "Add predefined audit columns with trigger" + augmap = {'schema sd': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_default': { + 'events': ['update'], 'level': 'row', + 'procedure': 'sd.audit_default', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function audit_default()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function audit_default()'][ + 'source'] == FUNC_SRC1 + + def test_nondefault_schema_with_trigger(self): + "Add predefined audit columns with trigger in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text)"] + augmap = {'schema s1': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_audit_default': { + 'events': ['update'], 'level': 'row', + 'procedure': 's1.audit_default', 'timing': 'before'}}} + assert expmap == dbmap['schema s1']['table t1'] + assert dbmap['schema s1']['function audit_default()']['returns'] == \ + 'trigger' + assert dbmap['schema s1']['function audit_default()'][ + 'source'] == FUNC_SRC1 + + def test_skip_existing_columns(self): + "Do not add already existing audit columns" + stmts = [CREATE_STMT, + "ALTER TABLE t1 ADD modified_by_user varchar(63) NOT NULL", + "ALTER TABLE t1 ADD modified_timestamp " + "timestamp with time zone NOT NULL"] + augmap = {'schema sd': {'table t1': { + 'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}] + assert expmap == dbmap['schema sd']['table t1']['columns'] + + def test_change_existing_columns(self): + "Change already existing audit columns" + stmts = [CREATE_STMT, "ALTER TABLE t1 ADD modified_by_user text ", + "ALTER TABLE t1 ADD modified_timestamp " + "timestamp with time zone NOT NULL"] + augmap = {'schema sd': {'table t1': {'audit_columns': 'default'}}} + dbmap = self.to_map(stmts, augmap) + expmap = [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}] + assert expmap == dbmap['schema sd']['table t1']['columns'] + + def test_custom_function_template(self): + "Add new (non-predefined) audit trigger using a function template" + template = """ + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + source = """ + BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + augmap = { + 'augmenter': { + 'audit_columns': {'custom': { + 'columns': ['modified_by_user', 'modified_timestamp'], + 'triggers': ['custom_audit']}}, + 'function_templates': {'custom_template': template}, + 'functions': {'custom_audit()': { + 'description': 'Maintain custom audit columns', + 'language': 'plpgsql', + 'returns': 'trigger', + 'security_definer': True, + 'source': '{{custom_template}}'}}, + 'triggers': {'custom_audit': { + 'events': ['insert', 'update'], + 'level': 'row', + 'name': '{{table_name}}_20_custom_audit', + 'procedure': 'custom_audit', + 'timing': 'before'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'custom'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_custom_audit': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.custom_audit', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function custom_audit()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function custom_audit()'][ + 'source'] == source + + def test_custom_function_inline_with_column_substitution(self): + "Add new (non-predefined) audit trigger using an inline definition" + template = """ + BEGIN + NEW.{{modified_by_user}} = SESSION_USER; + NEW.{{modified_timestamp}} = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + source = """ + BEGIN + NEW.modified_by_user = SESSION_USER; + NEW.modified_timestamp = CURRENT_TIMESTAMP::timestamp(0); + RETURN NEW; + END""" + augmap = { + 'augmenter': { + 'audit_columns': {'custom': { + 'columns': ['modified_by_user', 'modified_timestamp'], + 'triggers': ['custom_audit']}}, + 'functions': {'custom_audit()': { + 'description': 'Maintain custom audit columns', + 'language': 'plpgsql', + 'returns': 'trigger', + 'security_definer': True, + 'source': template}}, + 'triggers': {'custom_audit': { + 'events': ['insert', 'update'], + 'level': 'row', + 'name': '{{table_name}}_20_custom_audit', + 'procedure': 'custom_audit', + 'timing': 'before'}}}, + 'schema sd': {'table t1': { + 'audit_columns': 'custom'}}} + dbmap = self.to_map([CREATE_STMT], augmap) + expmap = {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'modified_by_user': {'type': 'character varying(63)', + 'not_null': True}}, + {'modified_timestamp': {'type': 'timestamp with time zone', + 'not_null': True}}], + 'triggers': {'t1_20_custom_audit': { + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.custom_audit', 'timing': 'before'}}} + assert expmap == dbmap['schema sd']['table t1'] + assert dbmap['schema sd']['function custom_audit()'][ + 'returns'] == 'trigger' + assert dbmap['schema sd']['function custom_audit()'][ + 'source'] == source diff --git a/tests/dbobject/__init__.py b/tests/dbobject/__init__.py new file mode 100644 index 0000000..b0bfa64 --- /dev/null +++ b/tests/dbobject/__init__.py @@ -0,0 +1 @@ +"""Pyrseas dbobject unit tests""" diff --git a/tests/dbobject/test_cast.py b/tests/dbobject/test_cast.py new file mode 100644 index 0000000..1b6af85 --- /dev/null +++ b/tests/dbobject/test_cast.py @@ -0,0 +1,171 @@ +# -*- coding: utf-8 -*- +"""Test casts""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +SOURCE = "SELECT CAST($1::int AS boolean)" +CREATE_FUNC = "CREATE FUNCTION int2_bool(smallint) RETURNS boolean " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE +CREATE_DOMAIN = "CREATE DOMAIN d1 AS integer" +CREATE_STMT1 = "CREATE CAST (smallint AS boolean) WITH FUNCTION " \ + "sd.int2_bool(smallint)" +CREATE_STMT3 = "CREATE CAST (d1 AS integer) WITH INOUT AS IMPLICIT" +DROP_STMT = "DROP CAST IF EXISTS (smallint AS boolean)" +COMMENT_STMT = "COMMENT ON CAST (smallint AS boolean) IS 'Test cast 1'" + + +class CastToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing casts""" + + def test_map_cast_function(self): + "Map a cast with a function" + dbmap = self.to_map([CREATE_FUNC, CREATE_STMT1], superuser=True) + expmap = {'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'} + assert dbmap['cast (smallint as boolean)'] == expmap + + def test_map_cast_inout(self): + "Map a cast with INOUT" + dbmap = self.to_map([CREATE_DOMAIN, CREATE_STMT3]) + expmap = {'context': 'implicit', 'method': 'inout', + 'depends_on': ['domain d1']} + assert dbmap['cast (sd.d1 as integer)'] == expmap + + def test_map_cast_comment(self): + "Map a cast comment" + dbmap = self.to_map([CREATE_FUNC, CREATE_STMT1, COMMENT_STMT], + superuser=True) + assert dbmap['cast (smallint as boolean)']['description'] == \ + 'Test cast 1' + + +class CastToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input casts""" + + def test_create_cast_function(self): + "Create a cast with a function" + stmts = [DROP_STMT, CREATE_FUNC] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == CREATE_STMT1 + + def test_create_cast_inout(self): + "Create a cast with INOUT" + stmts = [CREATE_DOMAIN, "DROP CAST IF EXISTS (d1 AS integer)"] + inmap = self.std_map() + inmap.update({'cast (d1 as integer)': { + 'context': 'implicit', 'method': 'inout'}}) + inmap['schema sd'].update({'domain d1': {'type': 'integer'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == CREATE_STMT3 + + def test_create_cast_schema(self): + "Create a cast using a type/domain in a non-default schema" + stmts = ["CREATE SCHEMA s1", "CREATE DOMAIN s1.d1 AS integer", + "DROP CAST IF EXISTS (integer AS s1.d1)"] + inmap = self.std_map() + inmap.update({'cast (integer as s1.d1)': { + 'context': 'assignment', 'method': 'binary coercible'}}) + inmap.update({'schema s1': {'domain d1': {'type': 'integer'}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "CREATE CAST (integer AS s1.d1) " \ + "WITHOUT FUNCTION AS ASSIGNMENT" + + def test_bad_cast_map(self): + "Error creating a cast with a bad map" + inmap = self.std_map() + inmap.update({'(smallint as boolean)': { + 'function': 'int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_cast(self): + "Drop an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP CAST (smallint AS boolean)" + + def test_cast_with_comment(self): + "Create a cast with a comment" + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Test cast 1', 'function': 'sd.int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + sql = self.to_sql(inmap, [DROP_STMT]) + # sql[0:1] -> SET, CREATE FUNCTION + assert fix_indent(sql[2]) == CREATE_STMT1 + assert sql[3] == COMMENT_STMT + + def test_comment_on_cast(self): + "Create a comment for an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Test cast 1', 'function': 'int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_cast_comment(self): + "Drop a comment on an existing cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1, COMMENT_STMT] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'function': 'int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + assert self.to_sql(inmap, stmts, superuser=True) == \ + ["COMMENT ON CAST (smallint AS boolean) IS NULL"] + + def test_change_cast_comment(self): + "Change existing comment on a cast" + stmts = [DROP_STMT, CREATE_FUNC, CREATE_STMT1, COMMENT_STMT] + inmap = self.std_map() + inmap.update({'cast (smallint as boolean)': { + 'description': 'Changed cast 1', 'function': 'int2_bool(smallint)', + 'context': 'explicit', 'method': 'function'}}) + inmap['schema sd'].update({'function int2_bool(smallint)': { + 'returns': 'boolean', 'language': 'sql', 'immutable': True, + 'source': SOURCE}}) + assert self.to_sql(inmap, stmts, superuser=True) == \ + ["COMMENT ON CAST (smallint AS boolean) IS 'Changed cast 1'"] + + def test_cast_function_view_depends(self): + "Cast that depends on a function that depends on a view. See #86" + stmts = ["CREATE TABLE t1 (id integer)"] + inmap = self.std_map() + inmap.update({'cast (sd.v1 as sd.t1)': { + 'context': 'explicit', 'function': 'sd.v1_to_t1(sd.v1)', + 'method': 'function'}}) + inmap['schema sd'].update({ + 'function v1_to_t1(sd.v1)': { + 'returns': 'sd.t1', 'language': 'plpgsql', + 'source': "\nDECLARE o sd.t1;\nBEGIN o:= ROW($1.id)::t1;\n" + "RETURN o;\nEND"}, + 'table t1': {'columns': [{'id': {'type': 'integer'}}]}, + 'view v1': {'definition': " SELECT t1.id\n FROM sd.t1;", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 3 + assert fix_indent(sql[0]) == \ + "CREATE VIEW sd.v1 AS SELECT t1.id FROM sd.t1" + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.v1_to_t1(sd.v1) " \ + "RETURNS sd.t1 LANGUAGE plpgsql AS $_$\nDECLARE o sd.t1;\n" \ + "BEGIN o:= ROW($1.id)::t1;\nRETURN o;\nEND$_$" + assert fix_indent(sql[2]) == "CREATE CAST (sd.v1 AS sd.t1) WITH " \ + "FUNCTION sd.v1_to_t1(sd.v1)" diff --git a/tests/dbobject/test_collation.py b/tests/dbobject/test_collation.py new file mode 100644 index 0000000..bcfa1ad --- /dev/null +++ b/tests/dbobject/test_collation.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +"""Test collations + +These tests require that the locale fr_FR.utf8 (or equivalent) be installed. +""" +import sys + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +if sys.platform == 'win32': + COLL = 'French_France.1252' +else: + COLL = 'fr_FR.UTF-8' + +CREATE_STMT = "CREATE COLLATION sd.c1 (LC_COLLATE = '%s', LC_CTYPE = '%s')" % ( + COLL, COLL) +COMMENT_STMT = "COMMENT ON COLLATION sd.c1 IS 'Test collation c1'" + + +class CollationToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing collations""" + + def test_map_collation1(self): + "Map a collation" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'lc_collate': COLL, 'lc_ctype': COLL} + assert dbmap['schema sd']['collation c1'] == expmap + + def test_map_collation_comment(self): + "Map a collation comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['collation c1']['description'] == \ + 'Test collation c1' + + def test_map_column_collation(self): + "Map a table with a column collation" + dbmap = self.to_map( + [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text COLLATE c1)"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text', 'collation': 'c1'}}], + 'depends_on': ['collation c1']} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_collation(self): + "Map an index with column collation" + stmts = [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE INDEX t1_idx ON t1 (c2 COLLATE c1)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': { + 'keys': [{'c2': {'collation': 'sd.c1'}}], + 'depends_on': ['collation c1']}}} + assert dbmap['schema sd']['table t1'] == expmap + + +class CollationToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input collations""" + + def test_create_collation1(self): + "Create a collation" + inmap = self.std_map() + inmap['schema sd'].update({'collation c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_collation_schema(self): + "Create a collation in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'collation c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE COLLATION s1.c1 (" \ + "LC_COLLATE = '%s', LC_CTYPE = '%s')" % (COLL, COLL) + + def test_bad_collation_map(self): + "Error creating a collation with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'c1': { + 'lc_collate': COLL, 'lc_ctype': COLL}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_collation(self): + "Drop an existing collation" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql[0] == "DROP COLLATION sd.c1" + + def test_collation_with_comment(self): + "Create a collation with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'collation c1': { + 'description': 'Test collation c1', + 'lc_collate': COLL, 'lc_ctype': COLL}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_create_table_column_collation(self): + "Create a table with a column with non-default collation" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text', 'collation': 'c1'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (" \ + 'c1 integer NOT NULL, c2 text COLLATE "c1")' + + def test_create_index_collation(self): + "Create an index with column collation" + stmts = [CREATE_STMT, "CREATE TABLE t1 (c1 integer, c2 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{'c2': {'collation': 'c1'}}]}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "CREATE INDEX t1_idx ON sd.t1 (c2 COLLATE c1)" + + def test_create_type_attribute_collation(self): + "Create a composite type with an attribute with non-default collation" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'text', 'collation': 'c1'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TYPE sd.t1 AS (x integer, " \ + 'y text COLLATE "c1")' diff --git a/tests/dbobject/test_column.py b/tests/dbobject/test_column.py new file mode 100644 index 0000000..d7b01d9 --- /dev/null +++ b/tests/dbobject/test_column.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +"""Test columns""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +TYPELIST = [ + ('SMALLINT', 'smallint'), + ('INTEGER', 'integer'), + ('BIGINT', 'bigint'), + ('int', 'integer'), + ('int2', 'smallint'), + ('int4', 'integer'), + ('int8', 'bigint'), + ('NUMERIC', 'numeric'), + ('NUMERIC(1)', 'numeric(1,0)'), + ('NUMERIC(12)', 'numeric(12,0)'), + ('NUMERIC(1000)', 'numeric(1000,0)'), + ('NUMERIC(12,2)', 'numeric(12,2)'), + ('NUMERIC(1000,500)', 'numeric(1000,500)'), + ('DECIMAL', 'numeric'), + ('dec(9,5)', 'numeric(9,5)'), + ('REAL', 'real'), + ('DOUBLE PRECISION', 'double precision'), + ('FLOAT', 'double precision'), + ('FLOAT(1)', 'real'), + ('FLOAT(24)', 'real'), + ('FLOAT(25)', 'double precision'), + ('FLOAT(53)', 'double precision'), + # SERIAL and BIGSERIAL have side effects + ('MONEY', 'money'), + ('CHARACTER(1)', 'character(1)'), + ('CHARACTER VARYING(200000)', 'character varying(200000)'), + ('CHAR(16)', 'character(16)'), + ('VARCHAR(256)', 'character varying(256)'), + ('TEXT', 'text'), + ('CHAR', 'character(1)'), + ('CHARACTER VARYING', 'character varying'), + ('"char"', '"char"'), + ('name', 'name'), + ('bytea', 'bytea'), + ('DATE', 'date'), + ('TIME', 'time without time zone'), + ('TIME WITHOUT TIME ZONE', 'time without time zone'), + ('TIME WITH TIME ZONE', 'time with time zone'), + ('TIMESTAMP', 'timestamp without time zone'), + ('TIMESTAMP WITHOUT TIME ZONE', 'timestamp without time zone'), + ('TIMESTAMP WITH TIME ZONE', 'timestamp with time zone'), + ('TIME(0)', 'time(0) without time zone'), + ('TIME(1) WITHOUT TIME ZONE', 'time(1) without time zone'), + ('TIME(2) WITH TIME ZONE', 'time(2) with time zone'), + ('TIMESTAMP(3)', 'timestamp(3) without time zone'), + ('TIMESTAMP(4) WITHOUT TIME ZONE', 'timestamp(4) without time zone'), + ('TIMESTAMP(5) WITH TIME ZONE', 'timestamp(5) with time zone'), + ('INTERVAL', 'interval'), + ('INTERVAL(6)', 'interval(6)'), + ('INTERVAL YEAR', 'interval year'), + ('INTERVAL MONTH', 'interval month'), + ('INTERVAL DAY', 'interval day'), + ('INTERVAL HOUR', 'interval hour'), + ('INTERVAL MINUTE', 'interval minute'), + ('INTERVAL SECOND', 'interval second'), + ('INTERVAL YEAR TO MONTH', 'interval year to month'), + ('INTERVAL DAY TO HOUR', 'interval day to hour'), + ('INTERVAL DAY TO MINUTE', 'interval day to minute'), + ('INTERVAL DAY TO SECOND', 'interval day to second'), + ('INTERVAL HOUR TO MINUTE', 'interval hour to minute'), + ('INTERVAL HOUR TO SECOND', 'interval hour to second'), + ('INTERVAL MINUTE TO SECOND', 'interval minute to second'), + ('INTERVAL SECOND(3)', 'interval second(3)'), + ('INTERVAL HOUR TO SECOND(5)', 'interval hour to second(5)'), + ('BOOLEAN', 'boolean'), + ('POINT', 'point'), + ('LINE', 'line'), + ('LSEG', 'lseg'), + ('BOX', 'box'), + ('PATH', 'path'), + ('POLYGON', 'polygon'), + ('CIRCLE', 'circle'), + ('cidr', 'cidr'), + ('inet', 'inet'), + ('macaddr', 'macaddr'), + ('BIT(2)', 'bit(2)'), + ('BIT VARYING(100)', 'bit varying(100)'), + ('BIT', 'bit(1)'), + ('BIT VARYING', 'bit varying'), + ('tsvector', 'tsvector'), + ('tsquery', 'tsquery'), + ('UUID', 'uuid'), + ('XML', 'xml'), + ('JSON', 'json'), + ('JSONB', 'jsonb')] + +CREATE_STMT1 = "CREATE TABLE t1 (c1 integer, c2 text)" +CREATE_STMT2 = "CREATE TABLE t1 (c1 integer, c2 text, c3 date)" +CREATE_STMT3 = "CREATE TABLE t1 (c1 integer, c2 text, c3 date, c4 text)" +DROP_COL_STMT = "ALTER TABLE t1 DROP COLUMN c3" + + +class ColumnToMapTestCase(DatabaseToMapTestCase): + """Test mapping of column-related elements in created tables""" + + def test_data_types(self): + "Map a table with columns for (almost) each native PostgreSQL type" + colstab = [] + colsmap = [] + for colnum, (coltype, maptype) in enumerate(TYPELIST): + col = "c%d" % (colnum + 1) + colstab.append("%s %s" % (col, coltype)) + if coltype == 'name' and self.db.version >= 120000: + colsmap.append({col: {'type': maptype, 'collation': 'C'}}) + else: + colsmap.append({col: {'type': maptype}}) + dbmap = self.to_map(["CREATE TABLE t1 (%s)" % ", ".join(colstab)]) + expmap = {'columns': colsmap} + assert dbmap['schema sd']['table t1'] == expmap + + def test_not_null(self): + "Map a table with a NOT NULL column" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 INTEGER NULL, " + "c3 INTEGER NOT NULL)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'integer', 'not_null': True}}]} + + assert dbmap['schema sd']['table t1'] == expmap + + def test_column_defaults(self): + "Map a table with various types and each with a DEFAULT clause" + stmts = ["CREATE TABLE t1 (c1 INTEGER DEFAULT 12345, " + "c2 NUMERIC DEFAULT 98.76, c3 REAL DEFAULT 15e-2, " + "c4 TEXT DEFAULT 'Abc def', c5 DATE DEFAULT CURRENT_DATE, " + "c6 TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, " + "c7 BOOLEAN DEFAULT FALSE)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [ + {'c1': {'type': 'integer', 'default': '12345'}}, + {'c2': {'type': 'numeric', 'default': '98.76'}}, + {'c3': {'type': 'real', 'default': '0.15'}}, + {'c4': {'type': 'text', 'default': "'Abc def'::text"}}, + {'c5': {'type': 'date', 'default': "CURRENT_DATE"}}, + {'c6': {'type': 'timestamp with time zone', 'default': + "CURRENT_TIMESTAMP"}}, + {'c7': {'type': 'boolean', 'default': 'false'}}]} + assert dbmap['schema sd']['table t1'] == expmap + + def test_statistics(self): + "Map a table with column statistics" + stmts = [CREATE_STMT1, "ALTER TABLE t1 ALTER c1 SET STATISTICS 100"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'statistics': 100}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_identity(self): + "Map a table with an IDENTITY column" + stmts = ["CREATE TABLE t1 (c1 integer GENERATED ALWAYS AS IDENTITY, " + "c2 text)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'identity': 'always'}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + expmap = {'data_type': 'integer', 'start_value': 1, 'increment_by': 1, + 'max_value': 2147483647, 'min_value': None, 'cache_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'} + assert dbmap['schema sd']['sequence t1_c1_seq'] == expmap + + +class ColumnToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of column-related statements from input schemas""" + + def test_create_table_with_defaults(self): + "Create a table with two column DEFAULTs, one referring to a SEQUENCE" + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'default': + "nextval('s1.t1_c1_seq'::regclass)"}}, + {'c2': {'type': 'text', 'not_null': True, + 'collation': 'en_US.utf8'}}, + {'c3': {'type': 'date', 'not_null': True, + 'default': "('now'::text)::date"}}]}, + 'sequence t1_c1_seq': { + 'cache_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE SEQUENCE s1.t1_c1_seq " \ + "START WITH 1 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" + assert fix_indent(sql[1]) == "CREATE TABLE s1.t1 (c1 integer " \ + "NOT NULL DEFAULT nextval('s1.t1_c1_seq'::regclass), c2 text " \ + 'NOT NULL COLLATE "en_US.utf8", c3 date NOT NULL ' \ + "DEFAULT ('now'::text)::date)" + assert sql[2] == "ALTER SEQUENCE s1.t1_c1_seq OWNED BY s1.t1.c1" + + def test_create_column_identity(self): + "Create a table with a column GENERATED AS IDENTITY" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True, + 'identity': 'by default'}}, + {'c2': {'type': 'text'}}]}, + 'sequence t1_c1_seq': { + 'cache_value': 1, 'data_type': 'integer', 'increment_by': 1, + 'max_value': None, 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}) + sql = self.to_sql(inmap) + assert len(sql) == 1 + assert fix_indent(sql[0]) == ( + "CREATE TABLE sd.t1 (c1 integer NOT NULL GENERATED BY DEFAULT AS " + "IDENTITY (SEQUENCE NAME sd.t1_c1_seq START WITH 1 INCREMENT BY 1 " + "NO MINVALUE NO MAXVALUE CACHE 1), c2 text)") + + def test_change_column_default(self): + "Change the default value for an existing column" + stmt = "CREATE TABLE t1 (c1 integer, c2 boolean default true)" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'boolean', 'default': 'false'}}]}}) + sql = self.to_sql(inmap, [stmt]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 SET DEFAULT false" + + def test_set_column_not_null(self): + "Change a nullable column to NOT NULL" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 SET NOT NULL" + + def test_change_column_types(self): + "Change the datatypes of two columns" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'bigint'}}, + {'c2': {'type': 'varchar(25)'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 TYPE bigint" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 TYPE varchar(25)" + + def test_add_column1(self): + "Add new column to a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}, {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_add_column2(self): + "Add column to a table that has a dropped column" + stmts = [CREATE_STMT2, "ALTER TABLE t1 DROP COLUMN c2"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c2 text" + + def test_add_column3(self): + "No change on a table that has a dropped column" + stmts = [CREATE_STMT3, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 0 + + def test_add_column4(self): + "Add two columns to a table that has a dropped column" + stmts = [CREATE_STMT2, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}, {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 date" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_drop_column1(self): + "Drop a column from the end of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c4" + + def test_drop_column2(self): + "Drop a column from the middle of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c3" + + def test_drop_column3(self): + "Drop a column from the beginning of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT3]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_rename_column(self): + "Rename a table column" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c3': {'type': 'text', 'oldname': 'c2'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert sql[0] == "ALTER TABLE sd.t1 RENAME COLUMN c2 TO c3" + + def test_drop_add_column1(self): + "Drop and re-add table column from the end, almost like a RENAME" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 date" + assert sql[1] == "ALTER TABLE sd.t1 DROP COLUMN c3" + + def test_drop_add_column2(self): + "Drop and re-add table column from the beginning" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + assert sql[1] == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_drop_add_column3(self): + "Drop and re-add table columns from table with dropped column" + stmts = [CREATE_STMT2, DROP_COL_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c2': {'type': 'text'}}, {'c3': {'type': 'date'}}, + {'c4': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 date" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + assert sql[2] == "ALTER TABLE sd.t1 DROP COLUMN c1" + + def test_drop_column_in_schema(self): + "Drop a column from a table in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text, c3 date)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE s1.t1 DROP COLUMN c3" + + def test_inherit_add_parent_column(self): + "Add a column to parent table, child should not add as well" + stmts = [CREATE_STMT1, "CREATE TABLE t2 (c3 date) INHERITS (t1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c4': {'type': 'text'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'date'}}, + {'c4': {'type': 'text', 'inherited': True}}], + 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c4 text" + + def test_inherit_drop_parent_column(self): + "Drop a column from a parent table, child should not drop as well" + stmts = [CREATE_STMT1, "CREATE TABLE t2 (c3 date) INHERITS (t1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c3': {'type': 'date'}}], 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 1 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 DROP COLUMN c2" + + def test_alter_statistics(self): + "Alter a table to add column statistics" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'statistics': 100}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT1, "ALTER TABLE t1 ALTER c2 " + "SET STATISTICS 1000"]) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c1 SET STATISTICS 100" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ALTER COLUMN c2 SET STATISTICS -1" diff --git a/tests/dbobject/test_constraint.py b/tests/dbobject/test_constraint.py new file mode 100644 index 0000000..16e25a1 --- /dev/null +++ b/tests/dbobject/test_constraint.py @@ -0,0 +1,1275 @@ +# -*- coding: utf-8 -*- +"""Test constraints""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +COMMENT_STMT = "COMMENT ON CONSTRAINT cns1 ON sd.t1 IS 'Test constraint cns1'" + + +class CheckConstraintToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created CHECK constraints""" + + def test_check_constraint_1(self): + "Map a table with a CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 SMALLINT CHECK (c2 < 1000))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'smallint'}}], + 'check_constraints': {'t1_c2_check': { + 'columns': ['c2'], 'expression': '(c2 < 1000)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_2(self): + "Map a table with a two-column, named CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, " + "CONSTRAINT t1_check_ratio CHECK (c2 * 100 / c1 <= 50))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}], + 'check_constraints': {'t1_check_ratio': { + 'columns': ['c2', 'c1'], + 'expression': '(((c2 * 100) / c1) <= 50)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_no_columns(self): + "Map a table with a check constraint without any columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER, " + "CONSTRAINT t1_empty CHECK (false))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}], + 'check_constraints': {'t1_empty': { + 'expression': 'false'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_check_constraint_inherited(self): + "Map a table with an inherited CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_c1_check " + "CHECK (c1 > 0))", "CREATE TABLE t2 (c2 text) INHERITS (t1)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text'}}], + 'inherits': ['t1'], + 'check_constraints': {'t1_c1_check': { + 'columns': ['c1'], 'inherited': True, + 'expression': '(c1 > 0)'}}} + assert dbmap['schema sd']['table t2'] == expmap + + +class CheckConstraintToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input CHECK constraints""" + + def test_create_w_check_constraint(self): + "Create new table with a single column CHECK constraint" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'check_constraints': { + 't1_c1_check': {'columns': ['c1'], + 'expression': '(c1 > 0 and c1 < 1000000)'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c1_check CHECK (c1 > 0 and c1 < 1000000)" + + def test_add_check_constraint(self): + "Add a two-column CHECK constraint to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'check_constraints': { + 't1_check_2_1': {'columns': ['c2', 'c1'], + 'expression': '(c2 != c1)'}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_check_2_1 CHECK (c2 != c1)" + + def test_add_check_constraint_no_columns(self): + "Add a CHECK constraint with no column" + stmts = ["CREATE TABLE t1 (c1 INTEGER)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], + 'check_constraints': { + 't1_empty': {'expression': 'false'}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_empty CHECK (false)" + + def test_add_check_inherited(self): + "Add a table with an inherited CHECK constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_c1_check " + "CHECK (c1 > 0))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], 'check_constraints': { + 't1_c1_check': {'columns': ['c1'], 'expression': '(c1 > 0)'}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text'}}], 'check_constraints': { + 't1_c1_check': + {'columns': ['c1'], 'expression': '(c1 > 0)', + 'inherited': True}}, 'inherits': ['t1']}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t2 (c2 text) " \ + "INHERITS (sd.t1)" + assert len(sql) == 1 + + def test_change_check_constraint(self): + "Change expression of a check constraint in an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER, CONSTRAINT t1_check1 " + "CHECK (c1 > 0), CONSTRAINT t1_check2 CHECK (c1 < 100))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}], 'check_constraints': { + 't1_check1': {'columns': ['c1'], 'expression': '(c1 > 10)'}, + 't1_check2': {'columns': ['c1'], 'expression': '(c1 < 100)'} + }} + }) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_check1" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_check1 CHECK (c1 > 10)" + assert len(sql) == 2 + + +class PrimaryKeyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created PRIMARY KEYs""" + + map_pkey1 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}} + map_pkey2 = {'columns': [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(5)', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}} + + map_pkey3 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_prim_key': {'columns': ['c1']}}} + + def test_primary_key_1(self): + "Map a table with a single-column primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey1 + + def test_primary_key_2(self): + "Map a table with a single-column primary key, table-level constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, PRIMARY KEY (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey1 + + def test_primary_key_3(self): + "Map a table with two-column primary key, atypical order" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT, " + "PRIMARY KEY (c2, c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey2 + + def test_primary_key_4(self): + "Map a table with a named primary key constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, " + "CONSTRAINT t1_prim_key PRIMARY KEY (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey3 + + def test_primary_key_5(self): + "Map a table with a named primary key, column level constraint" + stmts = ["CREATE TABLE t1 (" + "c1 INTEGER CONSTRAINT t1_prim_key PRIMARY KEY, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_pkey3 + + def test_primary_key_cluster(self): + "Map a table with a primary key and CLUSTER on it" + stmts = ["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CLUSTER t1 USING t1_pkey"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], 'cluster': True}}} + + def test_map_pk_comment(self): + "Map a primary key with a comment" + stmts = ["CREATE TABLE t1 (c1 integer CONSTRAINT cns1 PRIMARY KEY, " + "c2 text)", COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['primary_key']['cns1'][ + 'description'] == 'Test constraint cns1' + + +class PrimaryKeyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input PRIMARY KEYs""" + + def test_create_with_primary_key(self): + "Create new table with single column primary key" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'text'}}, {'c2': {'type': 'integer'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 text, c2 integer)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_add_primary_key(self): + "Add a two-column primary key to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1, c2)" + + def test_alter_primary_key_add(self): + "Add a two-column primary key to an existing table with primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 INTEGER, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1, c2)" + + def test_alter_primary_key_change(self): + "Change primary key" + stmts = ["CREATE TABLE t1 (c1 INTEGER PRIMARY KEY, c2 INTEGER, " + "c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_alter_primary_key_change_order(self): + "Change primary key" + stmts = ["CREATE TABLE t1 (c2 INTEGER, c1 INTEGER PRIMARY KEY)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2)" + + def test_drop_primary_key(self): + "Drop a primary key on an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL PRIMARY KEY, c2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}, + 'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey"] + + def test_primary_key_clustered(self): + "Create new table clustered on the primary key" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_pkey" + + def test_primary_key_uncluster(self): + "Remove cluster from table clustered on the primary key" + stmts = ["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CLUSTER t1 USING t1_pkey"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET WITHOUT CLUSTER" + + def test_change_primary_key(self): + "Changing primary key columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL," + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c1)" + assert len(sql) == 2 + + def test_change_order_primary_key(self): + "Changing primary key columns order" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL," + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_pkey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c2, c1)" + assert len(sql) == 2 + + +class ForeignKeyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created FOREIGN KEYs""" + + map_fkey1 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], + 'on_update': 'cascade', + 'on_delete': 'restrict', + 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc1'] + } + }}} + + map_fkey2 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + # PG 12 uses all column names to generate the constraint's name + map_fkey2_12 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_c3_c4_fkey': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + map_fkey3 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'foreign_keys': {'t1_fgn_key': { + 'columns': ['c2', 'c3', 'c4'], 'references': { + 'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}}} + + map_fkey4 = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(5)', + 'not_null': True}}, + {'c3': {'type': 'integer'}}, + {'c4': {'type': 'date'}}, + {'c5': {'type': 'text'}}], + 'primary_key': {'t1_prim_key': {'columns': ['c1', 'c2']}}, + 'foreign_keys': { + 't1_fgn_key1': { + 'columns': ['c2', 'c3', 'c4'], + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc2', 'pc1', 'pc3']}}, + 't1_fgn_key2': {'columns': ['c2'], + 'references': {'schema': 'sd', + 'table': 't3', + 'columns': ['qc1']}}}} + + def test_foreign_key_1(self): + "Map a table with a single-column foreign key on another table" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER REFERENCES t2 (pc1) " + "ON UPDATE CASCADE ON DELETE RESTRICT" + ", c3 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey1 + + def test_foreign_key_2(self): + "Map a table with a single-column foreign key, table level constraint" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, c3 TEXT, " + "FOREIGN KEY (c2) REFERENCES t2 (pc1)" + "ON UPDATE CASCADE ON DELETE RESTRICT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey1 + + def test_foreign_key_3(self): + "Map a table with a three-column foreign key" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "FOREIGN KEY (c2, c3, c4) REFERENCES t2 (pc2, pc1, pc3))"] + dbmap = self.to_map(stmts) + if self.db.version < 120000: + assert dbmap['schema sd']['table t1'] == self.map_fkey2 + else: + assert dbmap['schema sd']['table t1'] == self.map_fkey2_12 + + def test_foreign_key_4(self): + "Map a table with a named, three-column foreign key" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "CONSTRAINT t1_fgn_key FOREIGN KEY (c2, c3, c4) " + "REFERENCES t2 (pc2, pc1, pc3))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey3 + + def test_foreign_key_5(self): + "Map a table with a primary key and two foreign keys" + stmts = ["CREATE TABLE t2 (pc1 INTEGER, pc2 CHAR(5), pc3 DATE, " + "pc4 TEXT, PRIMARY KEY (pc2, pc1, pc3))", + "CREATE TABLE t3 (qc1 CHAR(5) PRIMARY KEY, qc2 text)", + "CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 INTEGER, " + "c4 DATE, c5 TEXT, " + "CONSTRAINT t1_prim_key PRIMARY KEY (c1, c2), " + "CONSTRAINT t1_fgn_key1 FOREIGN KEY (c2, c3, c4) " + "REFERENCES t2 (pc2, pc1, pc3), " + "CONSTRAINT t1_fgn_key2 FOREIGN KEY (c2) " + "REFERENCES t3 (qc1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_fkey4 + + def test_foreign_key_actions(self): + "Map a table with foreign key ON UPDATE/ON DELETE actions" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER, c3 TEXT, " + "FOREIGN KEY (c2) REFERENCES t2 (pc1) " + "ON UPDATE RESTRICT ON DELETE SET NULL)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'on_update': 'restrict', + 'on_delete': 'set null', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_cross_schema_foreign_key(self): + "Map a table with a foreign key on a table in another schema" + stmts = ["CREATE SCHEMA s1", "CREATE SCHEMA s2", + "CREATE TABLE s2.t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE s1.t1 (c1 INTEGER PRIMARY KEY, " + "c2 INTEGER REFERENCES s2.t2 (pc1), c3 TEXT)"] + dbmap = self.to_map(stmts) + t2map = {'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['pc1']}}} + t1map = {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer'}}, {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}, + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 's2', 'table': 't2', + 'columns': ['pc1']}}}}} + assert dbmap['schema s2']['table t2'] == t2map + assert dbmap['schema s1'] == t1map + + def test_multiple_foreign_key(self): + "Map a table with its primary key referenced by two others" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, " + "c2 integer REFERENCES t1 (pc1), c3 text, " + "c4 integer REFERENCES t1 (pc1))"] + dbmap = self.to_map(stmts) + t1map = {'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['pc1']}}} + t2map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}, + {'c4': {'type': 'integer'}}], + 'foreign_keys': { + 't2_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}, + 't2_c4_fkey': { + 'columns': ['c4'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == t1map + assert dbmap['schema sd']['table t2'] == t2map + + def test_foreign_key_dropped_column(self): + "Map a table with a foreign key after a column has been dropped" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, c2 text, c3 smallint, " + "c4 integer REFERENCES t1 (pc1))", + "ALTER TABLE t2 DROP COLUMN c3"] + dbmap = self.to_map(stmts) + t2map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c4': {'type': 'integer'}}], + 'foreign_keys': {'t2_c4_fkey': { + 'columns': ['c4'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t2'] == t2map + + def test_foreign_key_deferred(self): + "Check constraints deferred status" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, " + "c2 INTEGER REFERENCES t2 (pc1), " + "c3 INTEGER REFERENCES t2 (pc1) DEFERRABLE, " + "c4 INTEGER REFERENCES t2 (pc1) DEFERRABLE " + "INITIALLY DEFERRED)"] + dbmap = self.to_map(stmts) + fks = dbmap['schema sd']['table t1']['foreign_keys'] + assert not fks['t1_c2_fkey'].get('deferrable') + assert not fks['t1_c2_fkey'].get('deferred') + assert fks['t1_c3_fkey'].get('deferrable') + assert not fks['t1_c3_fkey'].get('deferred') + assert fks['t1_c4_fkey'].get('deferrable') + assert fks['t1_c4_fkey'].get('deferred') + + def test_map_foreign_key_match(self): + "Map a foreign key constraint with a MATCH specification" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, " + "c2 INTEGER REFERENCES t2 (pc1) MATCH FULL, c3 TEXT)"] + dbmap = self.to_map(stmts) + t1map = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}} + assert dbmap['schema sd']['table t1'] == t1map + + def test_map_fk_comment(self): + "Map a foreign key with a comment" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)", + "CREATE TABLE t1 (c1 INTEGER, c2 INTEGER " + "CONSTRAINT cns1 REFERENCES t2 (pc1), c3 TEXT)", COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['foreign_keys']['cns1'][ + 'description'] == 'Test constraint cns1' + + +class ForeignKeyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input FOREIGN KEYs""" + + def test_create_with_foreign_key(self): + "Create a table with a foreign key constraint" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap) + # can't control which table will be created first + crt1 = 0 + crt2 = 1 + if 't1' in sql[1]: + crt1 = 1 + crt2 = 0 + assert fix_indent(sql[crt1]) == \ + "CREATE TABLE sd.t1 (c11 integer, c12 text)" + assert fix_indent(sql[crt2]) == \ + "CREATE TABLE sd.t2 (c21 integer, c22 text, c23 integer)" + assert fix_indent(sql[2]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_pkey PRIMARY KEY (c11)" + assert fix_indent(sql[3]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + + def test_create_foreign_key_deferred(self): + "Create a table with various foreign key deferring constraints" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}], + 'unique_constraints': {'t1_c11_key': {'columns': ['c11']}}}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}, + {'c24': {'type': 'integer'}}, + {'c25': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}, + 't2_c24_fkey': { + 'columns': ['c24'], + 'references': {'columns': ['c11'], + 'table': 't1'}, + 'deferrable': True}, + 't2_c25_fkey': { + 'columns': ['c25'], + 'references': {'columns': ['c11'], + 'table': 't1'}, + 'deferrable': True, 'deferred': True}}}}) + sql = self.to_sql(inmap) + assert len(sql) == 6 + # can't control which table/constraint will be created first + sql[0:2] = list(sorted(sql[0:2])) + sql[2:5] = list(sorted(sql[2:5])) + assert fix_indent(sql[0]) == \ + "CREATE TABLE sd.t1 (c11 integer, c12 text)" + assert fix_indent(sql[1]) == "CREATE TABLE sd.t2 (c21 integer, " \ + "c22 text, c23 integer, c24 integer, c25 integer)" + assert fix_indent(sql[2]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c11_key UNIQUE (c11)" + assert fix_indent(sql[3]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + assert fix_indent(sql[4]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c24_fkey FOREIGN KEY (c24) REFERENCES sd.t1 (c11) DEFERRABLE" + assert fix_indent(sql[5]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c25_fkey FOREIGN KEY (c25) REFERENCES sd.t1 (c11) " \ + "DEFERRABLE INITIALLY DEFERRED" + + def test_add_foreign_key(self): + "Add a two-column foreign key to an existing table" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, " + "c12 INTEGER NOT NULL, c13 TEXT, PRIMARY KEY (c11, c12))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL, " + "c22 TEXT, c23 INTEGER, c24 INTEGER, PRIMARY KEY (c21))"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}, + {'c13': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11', 'c12']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}, + {'c24': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23', 'c24'], + 'references': {'columns': ['c11', 'c12'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23, c24) REFERENCES sd.t1 (c11, c12)" + + def test_alter_foreign_key1(self): + "Change foreign key: referencing column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY NOT NULL, " + "c12 INTEGER NOT NULL)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY NOT NULL, " + "c22 INTEGER, c23 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}, + {'c23': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c23'], + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11)" + + def test_alter_foreign_key_columns(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL UNIQUE, " + "c12 INTEGER NOT NULL UNIQUE)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY NOT NULL, " + "c22 INTEGER, c23 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'integer', 'not_null': True}}], + 'unique_constraints': { + 't1_c11_key': {'columns': ['c11']}, + 't1_c12_key': {'columns': ['c12']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}, + {'c23': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'references': {'columns': ['c12'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c12)" + + def test_alter_foreign_key_change_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11) ON UPDATE RESTRICT"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'on_update': 'cascade', + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE" + + def test_alter_foreign_key_add_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'on_update': 'cascade', + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE" + + def test_alter_foreign_key_drop_actions(self): + "Change foreign key: foreign column" + stmts = ["CREATE TABLE t1 (c11 INTEGER PRIMARY KEY)", + "CREATE TABLE t2 (c21 INTEGER PRIMARY KEY, c22 INTEGER)", + "ALTER TABLE t2 ADD CONSTRAINT t2_c22_fkey " + "FOREIGN KEY (c22) REFERENCES t1 (c11) ON UPDATE RESTRICT"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [ + {'c11': {'type': 'integer', 'not_null': True}}], + 'primary_key': { + 't1_pkey': {'columns': ['c11']}}}, + 'table t2': {'columns': [ + {'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}, + 'foreign_keys': {'t2_c22_fkey': { + 'columns': ['c22'], + 'references': {'columns': ['c11'], 'table': 't1'}, + }}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c22_fkey FOREIGN KEY (c22) REFERENCES sd.t1 (c11)" + + def test_drop_foreign_key(self): + "Drop a foreign key on an existing table" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, c12 TEXT, " + "PRIMARY KEY (c11))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL PRIMARY KEY, " + "c22 INTEGER NOT NULL REFERENCES t1 (c11), c23 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'integer', 'not_null': True}}, + {'c23': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey"] + + def test_drop_foreign_key_and_column(self): + "Drop a foreign key on an existing table and its referred column" + stmts = ["CREATE TABLE t1 (c11 INTEGER NOT NULL, c12 TEXT, " + "PRIMARY KEY (c11))", + "CREATE TABLE t2 (c21 INTEGER NOT NULL PRIMARY KEY, " + "c22 INTEGER NOT NULL REFERENCES t1 (c11), c23 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c11': {'type': 'integer', 'not_null': True}}, + {'c12': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c11']}}}, + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c23': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c22_fkey" + assert sql[1] == "ALTER TABLE sd.t2 DROP COLUMN c22" + + def test_create_foreign_key_actions(self): + "Create a table with foreign key ON UPDATE/ON DELETE actions" + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}]}, + 'table t2': {'columns': [{'c21': {'type': 'integer'}}, + {'c22': {'type': 'text'}}, + {'c23': {'type': 'integer'}}], + 'foreign_keys': {'t2_c23_fkey': { + 'columns': ['c23'], 'on_update': 'cascade', + 'on_delete': 'set default', + 'references': {'columns': ['c11'], + 'table': 't1'}}}}}) + sql = self.to_sql(inmap) + # won't check CREATE TABLEs explicitly here (see first test instead) + assert fix_indent(sql[2]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_c23_fkey FOREIGN KEY (c23) REFERENCES sd.t1 (c11) " \ + "ON UPDATE CASCADE ON DELETE SET DEFAULT" + + def test_foreign_key_match(self): + "Create a foreign key constraint with a MATCH specification" + stmts = ["CREATE TABLE t2 (pc1 INTEGER PRIMARY KEY, pc2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t1_c2_fkey': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't2', + 'columns': ['pc1']}}}}, + 'table t2': {'columns': [{'pc1': {'type': 'integer', + 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['pc1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_c2_fkey FOREIGN KEY (c2) REFERENCES sd.t2 (pc1) MATCH FULL" + + def test_change_columns_foreign_key(self): + "Changing foreign key columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1) " + "REFERENCES t1 (c1) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c2) REFERENCES sd.t1 (c1) MATCH FULL" + assert len(sql) == 2 + + def test_change_ref_columns_foreign_key(self): + "Changing foreign key reference columns" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1, c2) " + "REFERENCES t1 (c1, c2) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1', 'c2'], 'match': 'full', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c2', 'c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1, c2) REFERENCES sd.t1 (c2, c1) MATCH FULL" + + def test_change_match_foreign_key(self): + "Changing foreign key match" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1, c2))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, c2 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1, c2) " + "REFERENCES t1 (c1, c2) MATCH FULL)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1', 'c2']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1', 'c2'], 'match': 'simple', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1', 'c2']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1, c2) REFERENCES sd.t1 (c1, c2)" + assert len(sql) == 2 + + def test_change_actions_foreign_key(self): + "Changing foreign key actions" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "CONSTRAINT t1_pkey PRIMARY KEY (c1))", + "CREATE TABLE t2 (c1 INTEGER NOT NULL, " + "CONSTRAINT t2_fk FOREIGN KEY (c1) " + "REFERENCES t1 (c1) ON UPDATE CASCADE)"] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'table t2': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}], + 'foreign_keys': {'t2_fk': { + 'columns': ['c1'], 'on_update': 'restrict', + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['c1']}}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_fk" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t2 ADD CONSTRAINT " \ + "t2_fk FOREIGN KEY (c1) REFERENCES sd.t1 (c1) ON UPDATE RESTRICT" + assert len(sql) == 2 + + +class UniqueConstraintToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created UNIQUE constraints""" + + map_unique1 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1']}}} + + map_unique2 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'text'}}], + 'unique_constraints': { + 't1_c1_c2_key': {'columns': ['c1', 'c2']}}} + + map_unique3 = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unique_constraints': { + 't1_unique_key': {'columns': ['c1']}}} + + def test_unique_1(self): + "Map a table with a single-column unique constraint" + dbmap = self.to_map(["CREATE TABLE t1 (c1 INTEGER UNIQUE, c2 TEXT)"]) + assert dbmap['schema sd']['table t1'] == self.map_unique1 + + def test_unique_2(self): + "Map a table with a single-column unique constraint, table level" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, UNIQUE (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique1 + + def test_unique_3(self): + "Map a table with a two-column unique constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT, " + "UNIQUE (c1, c2))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique2 + + def test_unique_4(self): + "Map a table with a named unique constraint" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT, " + "CONSTRAINT t1_unique_key UNIQUE (c1))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique3 + + def test_unique_5(self): + "Map a table with a named unique constraint, column level" + stmts = ["CREATE TABLE t1 ( " + "c1 INTEGER CONSTRAINT t1_unique_key UNIQUE, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == self.map_unique3 + + def test_map_unique_cluster(self): + "Map a table with a unique constraint and CLUSTER on it" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text UNIQUE)", + "CLUSTER t1 USING t1_c2_key"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c2_key': { + 'columns': ['c2'], 'cluster': True}}} + + +class UniqueConstraintToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input UNIQUE constraints""" + + def test_create_w_unique_constraint(self): + "Create new table with a single column unique constraint" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c1_key UNIQUE (c1)" + + def test_add_unique_constraint(self): + "Add a two-column unique constraint to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'unique_constraints': {'t1_c2_key': {'columns': ['c2', 'c1'], + 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c2_key UNIQUE (c2, c1)" + + def test_alter_unique_constraint(self): + "Change unique constraint column" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL)", + "ALTER TABLE t1 ADD CONSTRAINT t1_ukey UNIQUE (c1)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}], + 'unique_constraints': {'t1_ukey': {'columns': ['c2'], + 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert fix_indent(sql[0]) == \ + "ALTER TABLE sd.t1 DROP CONSTRAINT t1_ukey" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_ukey UNIQUE (c2)" + + def test_drop_unique_constraint(self): + "Drop a unique constraint on an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL UNIQUE, c2 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}, + 'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["ALTER TABLE sd.t1 DROP CONSTRAINT t1_c1_key"] + + def test_create_unique_clustered(self): + "Create new table clustered on the unique constraint index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1'], + 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_c1_key" + + def test_unique_cluster(self): + "Cluster a table on the unique constraint index" + stmts = ["CREATE TABLE t1 (c1 integer UNIQUE, c2 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c1'], + 'cluster': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "CLUSTER sd.t1 USING t1_c1_key" + + def test_change_unique_constraint(self): + "Change columns in unique index" + stmts = ["CREATE TABLE t1 (c1 integer UNIQUE, c2 date)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'date'}}], + 'unique_constraints': {'t1_c1_key': {'columns': ['c2']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t1 DROP CONSTRAINT t1_c1_key" + assert sql[1] == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_c1_key UNIQUE (c2)" + assert len(sql) == 2 + + def test_change_order_unique_constraint(self): + "Change columns order in unique index" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, " + "CONSTRAINT t1_unique UNIQUE (c1, c2))"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'t1_unique': {'columns': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "ALTER TABLE sd.t1 DROP CONSTRAINT t1_unique" + assert sql[1] == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT t1_unique UNIQUE (c2, c1)" + assert len(sql) == 2 + + +class ConstraintCommentTestCase(InputMapToSqlTestCase): + """Test creation of comments on constraints""" + + def test_check_constraint_with_comment(self): + "Create a CHECK constraint with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'check_constraints': {'cns1': { + 'columns': ['c1'], 'expression': '(c1 > 50)', + 'description': 'Test constraint cns1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == \ + "ALTER TABLE sd.t1 ADD CONSTRAINT cns1 CHECK (c1 > 50)" + assert sql[2] == COMMENT_STMT + + def test_comment_on_primary_key(self): + "Create a comment for an existing primary key" + stmts = ["CREATE TABLE t1 (c1 text CONSTRAINT cns1 PRIMARY KEY, " + "c2 integer)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'text', 'not_null': True}}, + {'c2': {'type': 'integer'}}], + 'primary_key': {'cns1': {'columns': ['c1'], + 'description': 'Test constraint cns1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_foreign_key_comment(self): + "Drop the comment on an existing foreign key" + stmts = ["CREATE TABLE t2 (c21 integer PRIMARY KEY, c22 text)", + "CREATE TABLE t1 (c11 integer, c12 text, " + "c13 integer CONSTRAINT cns1 REFERENCES t2 (c21))", + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'table t2': { + 'columns': [{'c21': {'type': 'integer', 'not_null': True}}, + {'c22': {'type': 'text'}}], + 'primary_key': {'t2_pkey': {'columns': ['c21']}}}, + 'table t1': { + 'columns': [{'c11': {'type': 'integer'}}, + {'c12': {'type': 'text'}}, + {'c13': {'type': 'integer'}}], + 'foreign_keys': {'cns1': { + 'columns': ['c13'], + 'references': {'columns': ['c21'], 'table': 't2'}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONSTRAINT cns1 ON sd.t1 IS NULL"] + + def test_change_unique_constraint_comment(self): + "Change existing comment on a unique constraint" + stmts = ["CREATE TABLE t1 (c1 integer CONSTRAINT cns1 UNIQUE, " + "c2 text)", COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'unique_constraints': {'cns1': { + 'columns': ['c1'], + 'description': "Changed constraint cns1"}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONSTRAINT cns1 ON sd.t1 IS " + "'Changed constraint cns1'"] + + def test_constraint_comment_schema(self): + "Add comment on a constraint for a table in another schema" + stmts = ["CREATE SCHEMA s1", "CREATE TABLE s1.t1 (c1 integer " + "CONSTRAINT cns1 CHECK (c1 > 50), c2 text)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'check_constraints': {'cns1': { + 'columns': ['c1'], 'expression': '(c1 > 50)', + 'description': 'Test constraint cns1'}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "COMMENT ON CONSTRAINT cns1 ON s1.t1 IS " \ + "'Test constraint cns1'" diff --git a/tests/dbobject/test_conversion.py b/tests/dbobject/test_conversion.py new file mode 100644 index 0000000..303ee2b --- /dev/null +++ b/tests/dbobject/test_conversion.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +"""Test conversions""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE CONVERSION sd.c1 FOR 'LATIN1' TO 'UTF8' " \ + "FROM iso8859_1_to_utf8" +DROP_STMT = "DROP CONVERSION IF EXISTS sd.c1" +COMMENT_STMT = "COMMENT ON CONVERSION sd.c1 IS 'Test conversion c1'" + + +class ConversionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing conversions""" + + def test_map_conversion1(self): + "Map a conversion" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'} + assert dbmap['schema sd']['conversion c1'] == expmap + + def test_map_conversion_comment(self): + "Map a conversion comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['conversion c1']['description'] == \ + 'Test conversion c1' + + +class ConversionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input conversions""" + + def test_create_conversion(self): + "Create a conversion" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_conversion_schema(self): + "Create a conversion in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8', 'default': True}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE DEFAULT CONVERSION s1.c1 " \ + "FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8" + + def test_bad_conversion_map(self): + "Error creating a conversion with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_conversion(self): + "Drop an existing conversion" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql[0] == "DROP CONVERSION sd.c1" + + def test_conversion_with_comment(self): + "Create a conversion with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Test conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_conversion(self): + "Create a comment for an existing conversion" + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Test conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_conversion_comment(self): + "Drop a comment on an existing conversion" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON CONVERSION sd.c1 IS NULL"] + + def test_change_conversion_comment(self): + "Change existing comment on a conversion" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'conversion c1': { + 'description': 'Changed conversion c1', + 'source_encoding': 'LATIN1', 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8'}}) + sql = self.to_sql(inmap, stmts) + assert sql == [ + "COMMENT ON CONVERSION sd.c1 IS 'Changed conversion c1'"] diff --git a/tests/dbobject/test_domain.py b/tests/dbobject/test_domain.py new file mode 100644 index 0000000..35d3f7e --- /dev/null +++ b/tests/dbobject/test_domain.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +"""Test domains""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATEFUNC_STMT = ("CREATE FUNCTION sd.dc1(integer) RETURNS bool LANGUAGE sql " + "IMMUTABLE AS $_$select true$_$") +CREATE_STMT = "CREATE DOMAIN sd.d1 AS integer" +DROP_STMT = "DROP DOMAIN IF EXISTS d1" +COMMENT_STMT = "COMMENT ON DOMAIN d1 IS 'Test domain d1'" + + +class DomainToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created domains""" + + def test_domain(self): + "Map a simple domain" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['domain d1'] == {'type': 'integer'} + + def test_domain_not_null(self): + "Map a domain with a NOT NULL constraint" + dbmap = self.to_map([CREATE_STMT + " NOT NULL"]) + expmap = {'type': 'integer', 'not_null': True} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_default(self): + "Map a domain with a DEFAULT" + dbmap = self.to_map(["CREATE DOMAIN d1 AS date DEFAULT CURRENT_DATE"]) + expmap = {'type': 'date', 'default': 'CURRENT_DATE'} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_check(self): + "Map a domain with a CHECK constraint" + dbmap = self.to_map([CREATE_STMT + " CHECK (VALUE >= 1888)"]) + expmap = {'type': 'integer', 'check_constraints': { + 'd1_check': {'expression': '(VALUE >= 1888)'}}} + assert dbmap['schema sd']['domain d1'] == expmap + + def test_domain_depend_function(self): + "A domain is created after a function it depends on" + dbmap = self.to_map([CREATEFUNC_STMT, + CREATE_STMT + " CHECK (dc1(VALUE))"]) + expmap = {'type': 'integer', + 'check_constraints': { + 'd1_check': { + 'expression': 'sd.dc1(VALUE)', + 'depends_on': ['function dc1(integer)']}}} + assert dbmap['schema sd']['domain d1'] == expmap + + +class DomainToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input domains""" + + def test_create_domain_simple(self): + "Create a simple domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': {'type': 'integer'}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT] + + def test_create_domain_default(self): + "Create a domain with a DEFAULT and NOT NULL" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'not_null': True, 'default': 0}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT + " NOT NULL DEFAULT 0"] + + def test_create_domain_check(self): + "Create a domain with a CHECK constraint" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': {'d1_check': { + 'expression': '(VALUE >= 1888)'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert fix_indent(sql[1]) == "ALTER DOMAIN sd.d1 ADD CONSTRAINT " + \ + "d1_check CHECK (VALUE >= 1888)" + + def test_add_domain_check(self): + "Add a CHECK constraint to a domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': {'d1_check': { + 'expression': '(VALUE >= 1888)'}}}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "ALTER DOMAIN sd.d1 " + \ + "ADD CONSTRAINT d1_check CHECK (VALUE >= 1888)" + + def test_drop_domain(self): + "Drop an existing domain" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP DOMAIN sd.d1"] + + def test_rename_domain(self): + "Rename an existing domain" + inmap = self.std_map() + inmap['schema sd'].update({'domain d2': { + 'oldname': 'd1', 'type': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER DOMAIN sd.d1 RENAME TO d2"] + + def test_create_domain_before_function(self): + "Check that the domain is created after a function it depends on" + inmap = self.std_map() + inmap['schema sd'].update({'domain d1': { + 'type': 'integer', 'check_constraints': { + 'd1_check': {'expression': 'dc1(VALUE)', + 'depends_on': ['function dc1(integer)']}}}, + 'function dc1(integer)': { + 'language': 'sql', 'returns': 'bool', + 'source': 'select true', 'volatility': 'immutable'}}) + sql = self.to_sql(inmap) + assert len(sql) == 4 + assert sql[1] == CREATE_STMT + assert fix_indent(sql[2]) == CREATEFUNC_STMT + assert sql[3].startswith("ALTER DOMAIN sd.d1 ADD CONSTRAINT") diff --git a/tests/dbobject/test_eventtrig.py b/tests/dbobject/test_eventtrig.py new file mode 100644 index 0000000..a666cb1 --- /dev/null +++ b/tests/dbobject/test_eventtrig.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +"""Test event triggers""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +FUNC_SRC = "BEGIN RAISE NOTICE 'Command % executed', tg_tag; END" +CREATE_FUNC_STMT = "CREATE FUNCTION sd.f1() RETURNS event_trigger " \ + "LANGUAGE plpgsql AS $_$%s$_$" % FUNC_SRC +CREATE_STMT = "CREATE EVENT TRIGGER et1 ON ddl_command_end %s" \ + "EXECUTE PROCEDURE sd.f1()" +DROP_TABLE_STMT = "DROP TABLE IF EXISTS t1" +DROP_FUNC_STMT = "DROP FUNCTION IF EXISTS f1()" +COMMENT_STMT = "COMMENT ON EVENT TRIGGER et1 IS 'Test event trigger et1'" + + +class EventTriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing event triggers""" + + def test_map_event_trigger_simple(self): + "Map a simple event trigger" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1'] == { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()'} + + def test_map_event_trigger_filter(self): + "Map a trigger with tag filter variables" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ( + "WHEN tag IN ('CREATE TABLE', 'CREATE VIEW') ")] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1'] == { + 'enabled': True, 'event': 'ddl_command_end', + 'tags': ['CREATE TABLE', 'CREATE VIEW'], 'procedure': 'sd.f1()'} + + def test_map_event_trigger_comment(self): + "Map a trigger comment" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % '', COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['event trigger et1']['description'] == \ + 'Test event trigger et1' + + +class EventTriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_event_trigger_simple(self): + "Create a simple event trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % '' + + def test_create_event_trigger_filter(self): + "Create an event trigger with tag filter variables" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()', 'tags': ['CREATE TABLE', 'CREATE VIEW']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % ( + "WHEN tag IN ('CREATE TABLE', 'CREATE VIEW') ") + + def test_create_event_trigger_func_schema(self): + "Create an event trigger with function in a non-default schema" + inmap = self.std_map(plpgsql_installed=True) + inmap.update({'schema s1': {'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 's1.f1()'}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE FUNCTION s1.f1() " \ + "RETURNS event_trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_SRC + assert fix_indent(sql[1]) == "CREATE EVENT TRIGGER et1 " \ + "ON ddl_command_end EXECUTE PROCEDURE s1.f1()" + + def test_drop_event_trigger(self): + "Drop an existing event trigger" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP EVENT TRIGGER et1"] + + def test_drop_event_trigger_function(self): + "Drop an existing event trigger and the related function" + stmts = [CREATE_FUNC_STMT, CREATE_STMT % ''] + inmap = self.std_map(plpgsql_installed=True) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP EVENT TRIGGER et1" + assert sql[1] == "DROP FUNCTION sd.f1()" + + def test_create_event_trigger_with_comment(self): + "Create an event trigger with a comment" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'event_trigger', + 'source': FUNC_SRC}}) + inmap.update({'event trigger et1': { + 'enabled': True, 'event': 'ddl_command_end', + 'procedure': 'sd.f1()', 'description': 'Test event trigger et1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FUNC_STMT + assert fix_indent(sql[1]) == CREATE_STMT % '' + assert sql[2] == COMMENT_STMT diff --git a/tests/dbobject/test_extension.py b/tests/dbobject/test_extension.py new file mode 100644 index 0000000..d3a089f --- /dev/null +++ b/tests/dbobject/test_extension.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +"""Test extensions""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE EXTENSION pg_trgm" +TRGM_COMMENT = "text similarity measurement and index searching based on " \ + "trigrams" +PLR_DESCR = "load R interpreter and execute R script from within a database" + + +class ExtensionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing extensions""" + + superuser = True + def base_version(self): + if self.db.version < 110000: + return '1.3' + elif self.db.version < 120000: + return '1.4' + elif self.db.version < 150000: + return '1.5' + return '1.6' + + def test_map_extension(self): + "Map an existing extension" + VERS = self.base_version() + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['extension pg_trgm'] == { + 'schema': 'sd', 'version': VERS, 'description': TRGM_COMMENT} + + def test_map_no_depends(self): + "Ensure no dependencies are included when mapping an extension" + dbmap = self.to_map([CREATE_STMT]) + assert 'type gtrgm' not in dbmap['schema sd'] + assert 'operator %(text, text)' not in dbmap['schema sd'] + assert 'function show_trgm(text)' not in dbmap['schema sd'] + + def test_map_lang_extension(self): + "Map a procedural language as an extension" + dbmap = self.to_map(["CREATE EXTENSION plperl"]) + assert dbmap['extension plperl'] == { + 'schema': 'pg_catalog', 'version': '1.0', + 'description': "PL/Perl procedural language"} + assert 'language plperl' not in dbmap + + def test_map_extern_lang_extension(self): + "Map an externally maintained procedural language" + if self.db.version < 150000: + self.skipTest('Only available since PG 15') + dbmap = self.to_map(["CREATE EXTENSION plr"]) + assert dbmap['extension plr'] == { + 'schema': 'sd', 'version': '8.4.5', 'description': PLR_DESCR} + assert dbmap['language plr'] == {'trusted': False} + # Ensure we don't map core-maintained language + assert 'language plpgsql' not in dbmap + + def test_map_extension_schema(self): + "Map an existing extension" + VERS = self.base_version() + dbmap = self.to_map(["CREATE SCHEMA s1", CREATE_STMT + " SCHEMA s1"]) + assert dbmap['extension pg_trgm'] == { + 'schema': 's1', 'version': VERS, 'description': TRGM_COMMENT} + + def test_map_extension_plpython3u(self): + "Test a function created with extension other than plpgsql/plperl" + # See issue #103 + dbmap = self.to_map(["CREATE EXTENSION plpython3u", + "CREATE FUNCTION test() RETURNS int AS " + "'return 1' LANGUAGE plpython3u"]) + assert 'extension plpython3u' in dbmap + + +class ExtensionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input extensions""" + + def test_create_extension_simple(self): + "Create a extension that didn't exist" + inmap = self.std_map() + inmap.update({'extension pg_trgm': {'schema': 'sd'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + " SCHEMA sd" + + def test_bad_extension_map(self): + "Error creating a extension with a bad map" + inmap = self.std_map() + inmap.update({'pg_trgm': {'schema': 'sd'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_extension(self): + "Drop an existing extension" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP EXTENSION pg_trgm"] + + def test_create_extension_schema(self): + "Create a extension in a given schema" + inmap = self.std_map() + inmap.update({'schema s1': {}, + 'extension pg_trgm': {'schema': 's1', 'version': '1.0'}}) + sql = self.to_sql(inmap) + assert sql[0] == 'CREATE SCHEMA s1' + assert fix_indent(sql[1]) == \ + "CREATE EXTENSION pg_trgm SCHEMA s1 VERSION '1.0'" + + def test_create_lang_extension(self): + "Create a language extension and a function in that language" + inmap = self.std_map() + inmap.update({'extension plperl': {'schema': 'pg_catalog', + 'description': + "PL/Perl procedural language"}}) + inmap['schema sd'].update({'function f1()': { + 'language': 'plperl', 'returns': 'text', + 'source': "return \"dummy\";"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE EXTENSION plperl" + # skip over COMMENT statement + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE plperl AS $_$return \"dummy\";$_$" + + def test_create_extern_lang_extension(self): + "Create an externally maintained language together with a function" + inmap = self.std_map() + inmap.update({'extension plr': {'schema': 'sd', + 'description': PLR_DESCR}}) + inmap.update({'language plr': { 'trusted': 'false'}}) + inmap['schema sd'].update({'function f1()': { + 'language': 'plr', 'returns': 'text', + 'source': "return \"dummy\";"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE EXTENSION plr SCHEMA sd" + # skip over COMMENT statement + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE plr AS $_$return \"dummy\";$_$" + + def test_comment_extension(self): + "Change the comment for an existing extension" + inmap = self.std_map() + inmap.update({'extension pg_trgm': { + 'schema': 'sd', 'description': "Trigram extension"}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == ["COMMENT ON EXTENSION pg_trgm IS 'Trigram extension'"] + + def test_no_alter_owner_extension(self): + """Do not alter the owner of an existing extension. + + ALTER EXTENSION extension_name OWNER is not a valid form. + """ + # create a new owner that is different from self.db.user + new_owner = 'new_%s' % self.db.user + inmap = self.std_map() + inmap.update({'extension pg_trgm': {'schema': 'sd', + 'owner': new_owner}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert 'ALTER EXTENSION pg_trgm OWNER TO %s' % new_owner not in sql diff --git a/tests/dbobject/test_extern_file.py b/tests/dbobject/test_extern_file.py new file mode 100644 index 0000000..0e114f6 --- /dev/null +++ b/tests/dbobject/test_extern_file.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Test external files used in --multiple-files option""" +import sys + +import pytest + +from pyrseas.testutils import PyrseasTestCase +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.dbobject.schema import Schema +from pyrseas.dbobject.function import Function +from pyrseas.dbobject.table import Sequence, Table +from pyrseas.dbobject.view import View + +if sys.platform == 'win32': + COLL = 'French_France.1252' +else: + COLL = 'fr_FR.UTF-8' + +CREATE_FDW = "CREATE FOREIGN DATA WRAPPER " +SOURCE1 = "SELECT 'dummy'::text" +SOURCE2 = "SELECT $1::text" +DROP_TSC = "DROP TEXT SEARCH CONFIGURATION IF EXISTS tsc1, tsc2" +DROP_TSP = "DROP TEXT SEARCH PARSER IF EXISTS tsp1 CASCADE" + + +class ExternalFilenameMapTestCase(DatabaseToMapTestCase): + + def setUp(self): + super(ExternalFilenameMapTestCase, self).setUp() + self.remove_tempfiles() + + def test_map_casts(self): + "Map casts" + self.to_map(["CREATE FUNCTION int2_bool(smallint) RETURNS boolean " + "LANGUAGE sql IMMUTABLE AS " + "$_$SELECT CAST($1::int AS boolean)$_$", + "CREATE DOMAIN d1 AS integer", + "CREATE CAST (smallint AS boolean) WITH FUNCTION " + "int2_bool(smallint)", + "CREATE CAST (d1 AS integer) WITH INOUT AS IMPLICIT"], + superuser=True, multiple_files=True) + expmap = {'cast (smallint as boolean)': { + 'function': 'sd.int2_bool(smallint)', 'context': 'explicit', + 'method': 'function'}, 'cast (sd.d1 as integer)': + {'context': 'implicit', 'method': 'inout', + 'depends_on': ['domain d1']}} + assert self.yaml_load('cast.yaml') == expmap + + def test_map_extension(self): + "Map extensions" + TRGM_VERS = '1.6' + if self.db.version < 110000: + TRGM_VERS = '1.3' + elif self.db.version < 130000: + TRGM_VERS = '1.4' + elif self.db.version < 150000: + TRGM_VERS = '1.5' + self.to_map(["CREATE EXTENSION pg_trgm"], superuser=True, + multiple_files=True) + expmap = {'extension plpgsql': { + 'schema': 'pg_catalog', 'version': '1.0', + 'description': 'PL/pgSQL procedural language'}, + 'extension pg_trgm': {'schema': 'sd', 'version': TRGM_VERS, + 'description': "text similarity measurement " + "and index searching based on trigrams"}} + assert self.yaml_load('extension.yaml') == expmap + + def test_map_fd_wrappers(self): + "Map foreign data wrappers" + self.to_map([CREATE_FDW + "fdw1", CREATE_FDW + "fdw2 " + "OPTIONS (debug 'true')"], superuser=True, + multiple_files=True) + expmap = {'foreign data wrapper fdw1': {}, + 'foreign data wrapper fdw2': {'options': ['debug=true']}} + assert self.yaml_load('foreign_data_wrapper.yaml') == expmap + + def test_collations(self): + "Map collations" + self.to_map(["CREATE COLLATION coll1 (LC_COLLATE = '%s', " + "LC_CTYPE = '%s')" % (COLL, COLL), + "COMMENT ON COLLATION coll1 IS 'A test collation'", + "CREATE COLLATION coll2 (LC_COLLATE = '%s', " + "LC_CTYPE = '%s')" % (COLL, COLL)], + multiple_files=True) + assert self.yaml_load('collation.yaml', 'schema.sd') == { + 'collation coll1': {'lc_collate': COLL, 'lc_ctype': COLL, + 'description': "A test collation"}, + 'collation coll2': {'lc_collate': COLL, 'lc_ctype': COLL}} + + def test_map_conversion(self): + "Map conversions" + self.to_map(["CREATE CONVERSION conv1 FOR 'LATIN1' TO 'UTF8' " + "FROM iso8859_1_to_utf8", + "COMMENT ON CONVERSION conv1 IS 'A test conversion'"], + multiple_files=True) + assert self.yaml_load('conversion.yaml', 'schema.sd') == { + 'conversion conv1': {'source_encoding': 'LATIN1', + 'dest_encoding': 'UTF8', + 'function': 'iso8859_1_to_utf8', + 'description': 'A test conversion'}} + + def test_map_functions(self): + "Map functions" + self.to_map(["CREATE FUNCTION f1() RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE1, + "CREATE FUNCTION f2() RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE1], multiple_files=True) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable'} + assert self.yaml_load('function.f1.yaml', 'schema.sd') == { + 'function f1()': expmap} + assert self.yaml_load('function.f2.yaml', 'schema.sd') == { + 'function f2()': expmap} + + def test_map_functions_merged(self): + "Map functions into a merged file" + self.to_map(["CREATE FUNCTION f3(integer) RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE2, + "CREATE FUNCTION f3(real) RETURNS text LANGUAGE sql " + "IMMUTABLE AS $_$%s$_$" % SOURCE2], multiple_files=True) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE2, 'volatility': 'immutable'} + assert self.yaml_load('function.f3.yaml', 'schema.sd') == { + 'function f3(integer)': expmap, 'function f3(real)': expmap} + + def test_map_operator(self): + "Map operators and an operator class" + self.to_map(["CREATE OPERATOR < (PROCEDURE = int4lt, LEFTARG = int, " + "RIGHTARG = int)", "CREATE OPERATOR = (PROCEDURE = " + "int4eq, LEFTARG = int, RIGHTARG = int)", "CREATE " + "OPERATOR > (PROCEDURE = int4gt, LEFTARG = int, " + "RIGHTARG = int)", + "CREATE OPERATOR CLASS oc1 FOR TYPE integer USING btree " + "AS OPERATOR 1 sd.<, OPERATOR 3 sd.=, OPERATOR " + "5 sd.>, FUNCTION 1 btint4cmp(integer,integer)"], + superuser=True, multiple_files=True) + oprmap = {'operator <(integer, integer)': {'procedure': 'int4lt'}, + 'operator =(integer, integer)': {'procedure': 'int4eq'}, + 'operator >(integer, integer)': {'procedure': 'int4gt'}} + opcmap = {'operator class oc1 using btree': { + 'type': 'integer', 'operators': { + 1: 'sd.<(integer,integer)', 3: 'sd.=(integer,integer)', + 5: 'sd.>(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}} + assert self.yaml_load('operator.yaml', 'schema.sd') == oprmap + assert self.yaml_load('operator_class.yaml', 'schema.sd') == opcmap + + def test_map_operator_family(self): + "Map operator families" + self.to_map(["CREATE SCHEMA s1", + "CREATE OPERATOR FAMILY s1.of1 USING btree", + "CREATE OPERATOR FAMILY s1.of2 USING btree"], + superuser=True, multiple_files=True) + assert self.yaml_load('operator_family.yaml', 'schema.s1') == { + 'operator family of1 using btree': {}, + 'operator family of2 using btree': {}} + + def test_map_tables(self): + "Map tables" + self.to_map(["CREATE TABLE t1 (c1 integer PRIMARY KEY, c2 text)", + "CREATE TABLE t2 (c1 integer, c2 integer, c3 text, " + "FOREIGN KEY (c2) REFERENCES t1 (c1))"], + multiple_files=True) + expmap1 = {'table t1': {'columns': [ + {'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}} + expmap2 = {'table t2': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t2_c2_fkey': {'columns': ['c2'], 'references': { + 'schema': 'sd', 'table': 't1', 'columns': ['c1']}}}}} + assert self.yaml_load('table.t1.yaml', 'schema.sd') == expmap1 + assert self.yaml_load('table.t2.yaml', 'schema.sd') == expmap2 + + def test_map_tables_merged(self): + "Map tables into a merged file" + self.to_map(["CREATE TABLE account_transfers_with_extra_padding_1 " + "(c1 integer, c2 text)", + "CREATE TABLE account_transfers_with_extra_padding_2 " + "(c1 integer, c2 text)"], + multiple_files=True) + expmap = {'table account_transfers_with_extra_padding_1': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]}, + 'table account_transfers_with_extra_padding_2': {'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]}} + assert self.yaml_load('table.account_transfers_with_extra_pad.yaml', + 'schema.sd') == expmap + + def test_map_textsearch(self): + "Map text search components" + self.to_map([DROP_TSC, DROP_TSP, + "CREATE TEXT SEARCH PARSER tsp1 (START = prsd_start, " + "GETTOKEN = prsd_nexttoken, END = prsd_end, " + "LEXTYPES = prsd_lextype, HEADLINE = prsd_headline)", + "CREATE TEXT SEARCH CONFIGURATION tsc1 (PARSER = tsp1)", + "CREATE TEXT SEARCH CONFIGURATION tsc2 (PARSER = tsp1)"], + superuser=True, multiple_files=True) + assert self.yaml_load('text_search_configuration.yaml', + 'schema.sd') == \ + {'text search configuration tsc1': {'parser': 'tsp1'}, + 'text search configuration tsc2': {'parser': 'tsp1'}} + assert self.yaml_load('text_search_parser.yaml', 'schema.sd') == { + 'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}} + self.db.execute(DROP_TSC) + self.db.execute_commit(DROP_TSP) + + def test_map_drop_table(self): + "Map three tables, drop one and map the remaining two" + self.to_map(["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE TABLE t3 (c1 integer, c2 text)"], + multiple_files=True) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + for tbl in ['t1', 't2', 't3']: + assert self.yaml_load('table.%s.yaml' % tbl, 'schema.sd')[ + 'table %s' % tbl] == expmap + self.to_map(["DROP TABLE t2"], multiple_files=True) + with pytest.raises(IOError) as exc: + self.yaml_load('table.t2.yaml', 'schema.sd') + assert 'No such file' in str(exc.value) + for tbl in ['t1', 't3']: + assert self.yaml_load('table.%s.yaml' % tbl, 'schema.sd')[ + 'table %s' % tbl] == expmap + + +class ExternalFilenameTestCase(PyrseasTestCase): + + def test_function(self): + "Map a function" + obj = Function("Weird/Or-what?", 'sd', '', None, [], None, None, + None, None) + assert obj.extern_filename() == 'function.weird_or_what_.yaml' + + def test_schema(self): + "Map a schema" + obj = Schema(name="A/C Schema") + assert obj.extern_filename() == 'schema.a_c_schema.yaml' + + def test_long_name_schema(self): + "Map a schema with a long name" + nm = 'a_schema_with_a_very_but_very_very_long_long_long_loooonng_name' + obj = Schema(name=nm) + assert obj.extern_filename() == 'schema.%s.yaml' % nm + + def test_table(self): + "Map a table" + obj = Table("Weird/Or-what?HOW.WeiRD", '', None, None, []) + assert obj.extern_filename() == 'table.weird_or_what_how_weird.yaml' + + def test_table_unicode(self): + "Map a table with Unicode characters" + obj = Table("Fundação\\Größe таблица", '', None, None, []) + assert obj.extern_filename() == 'table.fundação_größe_таблица.yaml' + + def test_sequence(self): + "Map a sequence" + obj = Sequence("Weird/Or-what?_seq", '', None, None, []) + assert obj.extern_filename() == 'sequence.weird_or_what__seq.yaml' + + def test_view(self): + "Map a view" + obj = View("Weirder/Don't You Think?", '', None, None, [], '') + assert obj.extern_filename() == 'view.weirder_don_t_you_think_.yaml' diff --git a/tests/dbobject/test_foreign.py b/tests/dbobject/test_foreign.py new file mode 100644 index 0000000..ce9634c --- /dev/null +++ b/tests/dbobject/test_foreign.py @@ -0,0 +1,395 @@ +# -*- coding: utf-8 -*- +"""Test text search objects""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_FDW_STMT = "CREATE FOREIGN DATA WRAPPER fdw1" +CREATE_FS_STMT = "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1" +CREATE_UM_STMT = "CREATE USER MAPPING FOR PUBLIC SERVER fs1" +CREATE_FT_STMT = "CREATE FOREIGN TABLE sd.ft1 (c1 integer, c2 text) SERVER fs1" +DROP_FDW_STMT = "DROP FOREIGN DATA WRAPPER IF EXISTS fdw1" +DROP_FS_STMT = "DROP SERVER IF EXISTS fs1" +DROP_UM_STMT = "DROP USER MAPPING IF EXISTS FOR PUBLIC SERVER fs1" +COMMENT_FDW_STMT = "COMMENT ON FOREIGN DATA WRAPPER fdw1 IS " \ + "'Test foreign data wrapper fdw1'" +COMMENT_FS_STMT = "COMMENT ON SERVER fs1 IS 'Test server fs1'" + + +class ForeignDataWrapperToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign data wrappers""" + + superuser = True + + def test_map_fd_wrapper(self): + "Map an existing foreign data wrapper" + dbmap = self.to_map([CREATE_FDW_STMT]) + assert dbmap['foreign data wrapper fdw1'] == {} + + def test_map_wrapper_validator(self): + "Map a foreign data wrapper with a validator function" + dbmap = self.to_map(["CREATE FOREIGN DATA WRAPPER fdw1 " + "VALIDATOR postgresql_fdw_validator"]) + assert dbmap['foreign data wrapper fdw1'] == { + 'validator': 'postgresql_fdw_validator'} + + def test_map_wrapper_options(self): + "Map a foreign data wrapper with options" + dbmap = self.to_map(["CREATE FOREIGN DATA WRAPPER fdw1 " + "OPTIONS (debug 'true')"]) + assert dbmap['foreign data wrapper fdw1'] == { + 'options': ['debug=true']} + + def test_map_fd_wrapper_comment(self): + "Map a foreign data wrapper with a comment" + dbmap = self.to_map([CREATE_FDW_STMT, COMMENT_FDW_STMT]) + assert dbmap['foreign data wrapper fdw1']['description'] == \ + 'Test foreign data wrapper fdw1' + + +class ForeignDataWrapperToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign data wrappers""" + + def test_create_fd_wrapper(self): + "Create a foreign data wrapper that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + + def test_create_wrapper_validator(self): + "Create a foreign data wrapper with a validator function" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'validator': 'postgresql_fdw_validator'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FOREIGN DATA WRAPPER fdw1 " \ + "VALIDATOR postgresql_fdw_validator" + + def test_create_wrapper_options(self): + "Create a foreign data wrapper with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'options': ['debug=true']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FOREIGN DATA WRAPPER fdw1 " \ + "OPTIONS (debug 'true')" + + def test_bad_map_fd_wrapper(self): + "Error creating a foreign data wrapper with a bad map" + inmap = self.std_map() + inmap.update({'fdw1': {}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_fd_wrapper(self): + "Drop an existing foreign data wrapper" + sql = self.to_sql(self.std_map(), [CREATE_FDW_STMT], superuser=True) + assert sql[0] == "DROP FOREIGN DATA WRAPPER fdw1" + + def test_alter_wrapper_options(self): + "Change foreign data wrapper options" + stmts = [CREATE_FDW_STMT + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'options': ['opt1=valX', 'opt3=valY']}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER FOREIGN DATA WRAPPER fdw1 " + assert sorted(sql[1][1:-1].split(', ')) == [ + "DROP opt2", "SET opt1 'valX'", "opt3 'valY'"] + + def test_comment_on_fd_wrapper(self): + "Create a comment for an existing foreign data wrapper" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'description': "Test foreign data wrapper fdw1"}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert sql[0] == COMMENT_FDW_STMT + + +class ForeignServerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign servers""" + + superuser = True + + def test_map_server(self): + "Map an existing foreign server" + dbmap = self.to_map([CREATE_FDW_STMT, CREATE_FS_STMT]) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': {}} + + def test_map_server_type_version(self): + "Map a foreign server with type and version" + stmts = [CREATE_FDW_STMT, + "CREATE SERVER fs1 TYPE 'test' VERSION '1.0' " + "FOREIGN DATA WRAPPER fdw1"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'type': 'test', 'version': '1.0'}} + + def test_map_server_options(self): + "Map a foreign server with options" + stmts = [CREATE_FDW_STMT, + "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1 " + "OPTIONS (dbname 'test')"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'options': ['dbname=test']}} + + def test_map_server_comment(self): + "Map a foreign server with a comment" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, COMMENT_FS_STMT] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1'] == {'server fs1': { + 'description': 'Test server fs1'}} + + +class ForeignServerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign servers""" + + def test_create_server(self): + "Create a foreign server that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == CREATE_FS_STMT + + def test_create_wrapper_server(self): + "Create a foreign data wrapper and its server" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + assert fix_indent(sql[1]) == CREATE_FS_STMT + + def test_create_server_type_version(self): + "Create a foreign server with type and version" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'type': 'test', 'version': '1.0'}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE SERVER fs1 TYPE 'test' " \ + "VERSION '1.0' FOREIGN DATA WRAPPER fdw1" + + def test_create_server_options(self): + "Create a foreign server with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'options': ['dbname=test']}}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE SERVER fs1 " \ + "FOREIGN DATA WRAPPER fdw1 OPTIONS (dbname 'test')" + + def test_drop_server(self): + "Drop an existing foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql[0] == "DROP SERVER fs1" + + def test_add_server_options(self): + "Add options to a foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'options': ['opt1=valA', 'opt2=valB']}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER SERVER fs1 " + assert sorted(sql[1][1:-1].split(', ')) == [ + "opt1 'valA'", "opt2 'valB'"] + + def test_drop_server_wrapper(self): + "Drop an existing foreign data wrapper and its server" + sql = self.to_sql(self.std_map(), [CREATE_FDW_STMT, CREATE_FS_STMT], + superuser=True) + assert sql[0] == "DROP SERVER fs1" + assert sql[1] == "DROP FOREIGN DATA WRAPPER fdw1" + + def test_comment_on_server(self): + "Create a comment for an existing foreign server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'description': "Test server fs1"}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_FS_STMT] + + +class UserMappingToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing user mappings""" + + superuser = True + + def test_map_user_mapping(self): + "Map an existing user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1']['server fs1'][ + 'user mappings'] == {'PUBLIC': {}} + + def test_map_user_mapping_options(self): + "Map a user mapping with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT + + " OPTIONS (user 'john', password 'doe')"] + dbmap = self.to_map(stmts) + assert dbmap['foreign data wrapper fdw1']['server fs1'] == { + 'user mappings': {'PUBLIC': {'options': [ + 'password=doe', 'user=john']}}} + + +class UserMappingToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input user mappings""" + + def test_create_user_mapping(self): + "Create a user mapping that didn't exist" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert fix_indent(sql[0]) == CREATE_UM_STMT + + def test_create_wrapper_server_mapping(self): + "Create a FDW, server and user mapping" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_FDW_STMT + assert fix_indent(sql[1]) == CREATE_FS_STMT + assert fix_indent(sql[2]) == CREATE_UM_STMT + + def test_create_user_mapping_options(self): + "Create a user mapping with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {'options': [ + 'user=john', 'password=doe']}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert fix_indent(sql[0]) == CREATE_UM_STMT + \ + " OPTIONS (user 'john', password 'doe')" + + def test_drop_user_mapping(self): + "Drop an existing user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_UM_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP USER MAPPING FOR PUBLIC SERVER fs1" + + def test_drop_user_mapping_options(self): + "Drop options from a user mapping" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, + CREATE_UM_STMT + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'user mappings': {'PUBLIC': {}}}}}) + sql = self.to_sql(inmap, stmts, superuser=True) + sql = fix_indent(sql[0]).split('OPTIONS ') + assert sql[0] == "ALTER USER MAPPING FOR PUBLIC SERVER fs1 " + assert sorted(sql[1][1:-1].split(', ')) == ["DROP opt1", "DROP opt2"] + + +class ForeignTableToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing foreign tables""" + + superuser = True + + def test_map_foreign_table(self): + "Map an existing foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1'} + assert dbmap['schema sd']['foreign table ft1'] == expmap + + def test_map_foreign_table_options(self): + "Map a foreign table with options" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT + + " OPTIONS (user 'jack')"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'options': ['user=jack']} + assert dbmap['schema sd']['foreign table ft1'] == expmap + + +class ForeignTableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input foreign tables""" + + superuser = True + + def test_create_foreign_table(self): + "Create a foreign table that didn't exist" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1'}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT, CREATE_FS_STMT]) + assert fix_indent(sql[0]) == CREATE_FT_STMT + + def test_create_foreign_table_options(self): + "Create a foreign table with options" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'options': ['user=jack']}}) + sql = self.to_sql(inmap, [CREATE_FDW_STMT, CREATE_FS_STMT]) + assert fix_indent(sql[0]), CREATE_FT_STMT + " OPTIONS (user 'jack')" + + def test_bad_map_foreign_table(self): + "Error creating a foreign table with a bad map" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_foreign_table(self): + "Drop an existing foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP FOREIGN TABLE sd.ft1" + + def test_drop_foreign_table_server(self): + "Drop a foreign table and associated server" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP FOREIGN TABLE sd.ft1" + assert sql[1] == "DROP SERVER fs1" + + def test_alter_foreign_table_options(self): + "Alter options for a foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT + + " OPTIONS (opt1 'valA', opt2 'valB')"] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'options': ['opt1=valX', 'opt2=valY']}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]), "ALTER FOREIGN TABLE ft1 " \ + "OPTIONS (SET opt1 'valX', SET opt2 'valY')" + + def test_add_column(self): + "Add new column to a foreign table" + stmts = [CREATE_FDW_STMT, CREATE_FS_STMT, CREATE_FT_STMT] + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], 'server': 'fs1'}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == \ + "ALTER FOREIGN TABLE sd.ft1 ADD COLUMN c3 date" diff --git a/tests/dbobject/test_function.py b/tests/dbobject/test_function.py new file mode 100644 index 0000000..42d0446 --- /dev/null +++ b/tests/dbobject/test_function.py @@ -0,0 +1,522 @@ +# -*- coding: utf-8 -*- +"""Test functions""" + +import pytest + +from inspect import cleandoc + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +SOURCE1 = "SELECT 'dummy'::text" +CREATE_STMT1 = "CREATE FUNCTION sd.f1() RETURNS text LANGUAGE sql IMMUTABLE " \ + "AS $_$%s$_$" % SOURCE1 + +SOURCE2 = "SELECT GREATEST($1, $2)" +CREATE_STMT2 = "CREATE FUNCTION sd.f1(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE2 +COMMENT_STMT = "COMMENT ON FUNCTION sd.f1(integer, integer) IS " \ + "'Test function f1'" + +SOURCE3 = "SELECT * FROM generate_series($1, $2)" +CREATE_STMT3 = "CREATE FUNCTION f2(integer, integer) RETURNS SETOF integer " \ + "ROWS 20 LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE3 + +SOURCE4 = "SELECT $1 + $2" +CREATE_STMT4 = "CREATE FUNCTION fadd(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE4 + +SOURCE5 = "SELECT $1 - $2" +CREATE_STMT5 = "CREATE FUNCTION fsub(integer, integer) RETURNS integer " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE5 + + +class FunctionToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing functions""" + + def test_map_function1(self): + "Map a very simple function with no arguments" + dbmap = self.to_map([CREATE_STMT1]) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable'} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_function_with_args(self): + "Map a function with two arguments" + stmts = ["CREATE FUNCTION f1(integer, integer) RETURNS integer " + "LANGUAGE sql AS $_$%s$_$" % SOURCE2] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2} + + def test_map_function_default_args(self): + "Map a function with default arguments" + stmts = ["CREATE FUNCTION f1(integer, integer) RETURNS integer " + "LANGUAGE sql AS $_$%s$_$" % SOURCE2] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2} + + def test_map_void_function(self): + "Map a function returning void" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE FUNCTION f1() RETURNS void LANGUAGE sql AS " + "$_$INSERT INTO t1 VALUES (1, 'dummy')$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'void', + 'source': "INSERT INTO t1 VALUES (1, 'dummy')"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_setof_row_function(self): + "Map a function returning a set of rows" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE FUNCTION f1() RETURNS SETOF t1 LANGUAGE sql AS " + "$_$SELECT * FROM t1$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'SETOF sd.t1', + 'source': "SELECT * FROM t1"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_security_definer_function(self): + "Map a function that is SECURITY DEFINER" + stmts = ["CREATE FUNCTION f1() RETURNS text LANGUAGE sql " + "SECURITY DEFINER AS $_$%s$_$" % SOURCE1] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'security_definer': True} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_c_lang_function(self): + "Map a dynamically loaded C language function" + # NOTE 1: Needs contrib/spi module to be available + # NOTE 2: Needs superuser privilege + stmts = ["CREATE FUNCTION autoinc() RETURNS trigger " + "AS '$libdir/autoinc' LANGUAGE c"] + dbmap = self.to_map(stmts, superuser=True) + expmap = {'language': 'c', 'obj_file': '$libdir/autoinc', + 'link_symbol': 'autoinc', 'returns': 'trigger'} + assert dbmap['schema sd']['function autoinc()'] == expmap + + def test_map_function_config(self): + "Map a function with a configuration parameter" + stmts = ["CREATE FUNCTION f1() RETURNS date LANGUAGE sql SET " + "datestyle to postgres, dmy AS $_$SELECT CURRENT_DATE$_$"] + dbmap = self.to_map(stmts) + expmap = {'language': 'sql', 'returns': 'date', + 'configuration': ['DateStyle=postgres, dmy'], + 'source': "SELECT CURRENT_DATE"} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_function_comment(self): + "Map a function comment" + dbmap = self.to_map([CREATE_STMT2, COMMENT_STMT]) + assert dbmap['schema sd']['function f1(integer, integer)'][ + 'description'] == 'Test function f1' + + def test_map_function_rows(self): + "Map a function rows" + dbmap = self.to_map([CREATE_STMT3]) + assert dbmap['schema sd']['function f2(integer, integer)'][ + 'rows'] == 20 + + def test_map_function_leakproof(self): + "Map a function with LEAKPROOF qualifier" + stmt = CREATE_STMT4.replace("IMMUTABLE", "IMMUTABLE LEAKPROOF") + dbmap = self.to_map([stmt], superuser=True) + expmap = {'language': 'sql', 'returns': 'integer', 'leakproof': True, + 'source': SOURCE4, 'volatility': 'immutable'} + assert dbmap['schema sd']['function fadd(integer, integer)'] == \ + expmap + + +class FunctionToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input functions""" + + def test_create_function1(self): + "Create a very simple function with no arguments" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == CREATE_STMT1 + + def test_create_function_with_args(self): + "Create a function with two arguments" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + + def test_create_setof_row_function(self): + "Create a function returning a set of rows" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({ + 'function f1()': {'language': 'sql', 'returns': 'SETOF t1', + 'source': "SELECT * FROM t1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "SETOF t1 LANGUAGE sql AS $_$SELECT * FROM t1$_$" + + def test_create_setof_row_function_rows(self): + "Create a function returning a set of rows with suggested number" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({ + 'function f1()': {'language': 'sql', 'returns': 'SETOF t1', + 'source': "SELECT * FROM t1", 'rows': 50}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE FUNCTION sd.f1() RETURNS SETOF " \ + "t1 LANGUAGE sql ROWS 50 AS $_$SELECT * FROM t1$_$" + + def test_create_security_definer_function(self): + "Create a SECURITY DEFINER function" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'security_definer': True}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS text " \ + "LANGUAGE sql SECURITY DEFINER AS $_$%s$_$" % SOURCE1 + + def test_create_c_lang_function(self): + "Create a dynamically loaded C language function" + # NOTE 1: Needs contrib/spi module to be available + # NOTE 2: Needs superuser privilege + inmap = self.std_map() + inmap['schema sd'].update({'function autoinc()': { + 'language': 'c', 'returns': 'trigger', + 'obj_file': '$libdir/autoinc'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE FUNCTION sd.autoinc() " \ + "RETURNS trigger LANGUAGE c AS '$libdir/autoinc', 'autoinc'" + + def test_create_function_config(self): + "Create a function with a configuration parameter" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'date', + 'configuration': ['DateStyle=postgres, dmy'], + 'source': "SELECT CURRENT_DATE"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS date " \ + "LANGUAGE sql SET DateStyle=postgres, dmy AS " \ + "$_$SELECT CURRENT_DATE$_$" + + def test_create_function_in_schema(self): + "Create a function within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[1]) == "CREATE FUNCTION s1.f1() RETURNS text " \ + "LANGUAGE sql IMMUTABLE AS $_$%s$_$" % SOURCE1 + + def test_bad_function_map(self): + "Error creating a function with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_function1(self): + "Drop an existing function with no arguments" + sql = self.to_sql(self.std_map(), [CREATE_STMT1]) + assert sql == ["DROP FUNCTION sd.f1()"] + + def test_drop_function_with_args(self): + "Drop an existing function which has arguments" + sql = self.to_sql(self.std_map(), [CREATE_STMT2]) + assert sql == ["DROP FUNCTION sd.f1(integer, integer)"] + + def test_change_function_defn(self): + "Change function definition" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', + 'source': "SELECT 'example'::text", 'volatility': 'immutable'}}) + sql = self.to_sql(inmap, [CREATE_STMT1]) + assert fix_indent(sql[1]) == "CREATE OR REPLACE FUNCTION sd.f1() " \ + "RETURNS text LANGUAGE sql IMMUTABLE AS " \ + "$_$SELECT 'example'::text$_$" + + def test_function_with_comment(self): + "Create a function with a comment" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Test function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + assert sql[2] == COMMENT_STMT + + def test_comment_on_function(self): + "Create a comment for an existing function" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Test function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert sql == [COMMENT_STMT] + + def test_drop_function_comment(self): + "Drop a comment on an existing function" + stmts = [CREATE_STMT2, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON FUNCTION sd.f1(integer, integer) IS NULL"] + + def test_change_function_comment(self): + "Change existing comment on a function" + stmts = [CREATE_STMT2, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'description': 'Changed function f1', 'language': 'sql', + 'returns': 'integer', 'source': SOURCE2}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON FUNCTION sd.f1(integer, integer) IS " + "'Changed function f1'"] + + def test_function_leakproof(self): + "Create a function with LEAKPROOF qualifier" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'leakproof': True, + 'source': SOURCE4, 'volatility': 'immutable'}}) + sql = self.to_sql(inmap, superuser=True) + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1(integer, integer)"\ + " RETURNS integer LANGUAGE sql IMMUTABLE LEAKPROOF AS " \ + "$_$%s$_$" % SOURCE4 + + def test_alter_function_leakproof(self): + "Change a function with LEAKPROOF qualifier" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function fadd(integer, integer)': { + 'language': 'sql', 'returns': 'integer', + 'source': SOURCE4, 'volatility': 'immutable'}}) + stmt = CREATE_STMT4.replace("IMMUTABLE", "IMMUTABLE LEAKPROOF") + sql = self.to_sql(inmap, [stmt], superuser=True) + assert fix_indent(sql[0]) == \ + "ALTER FUNCTION sd.fadd(integer, integer) NOT LEAKPROOF" + + def test_change_function_return_type(self): + source = lambda rtype: "SELECT '127.0.0.1'::{}".format(rtype) + old_type = 'text' + new_type = 'inet' + statement = lambda rtype: cleandoc(""" + CREATE OR REPLACE FUNCTION sd.fget_addr() + RETURNS {rtype} + LANGUAGE sql + IMMUTABLE + AS $_${body}$_$""" + ).format( + rtype=rtype, + body=source(rtype), + ).replace('\n', ' ') + + inmap = self.std_map() + inmap['schema sd'].update({ + 'function fget_addr()': { + 'language': 'sql', + 'returns': new_type, + 'source': source(new_type), + } + }) + sql = self.to_sql(inmap, [statement(old_type)]) + assert statement(new_type) == fix_indent(sql[1]) + + +class AggregateToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing aggregates""" + + def test_map_aggregate_simple(self): + "Map a simple aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer'} + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_aggregate_init_final(self): + "Map an aggregate with an INITCOND and a FINALFUNC" + stmts = [CREATE_STMT2, + "CREATE FUNCTION f2(integer) RETURNS float " + "LANGUAGE sql AS $_$SELECT $1::float$_$ IMMUTABLE", + "CREATE AGGREGATE a1 (integer) (SFUNC = f1, STYPE = integer, " + "FINALFUNC = f2, INITCOND = '-1')"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', + 'initcond': '-1', 'finalfunc': 'f2'} + assert dbmap['schema sd']['function f1(integer, integer)'] == \ + {'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'} + assert dbmap['schema sd']['function f2(integer)'] == \ + {'language': 'sql', 'returns': 'double precision', + 'source': "SELECT $1::float", 'volatility': 'immutable'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_aggregate_sortop(self): + "Map an aggregate with a SORTOP" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer, SORTOP = >)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', + 'sortop': 'pg_catalog.>'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_moving_aggregate(self): + "Map a moving-aggregate mode function" + stmts = [CREATE_STMT4, CREATE_STMT5, + "CREATE AGGREGATE a1 (integer) (sfunc = fadd, " + "stype = integer, initcond = '0', msfunc = fadd, " + "minvfunc = fsub, mstype = integer, minitcond = '0')"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'fadd', 'stype': 'integer', 'initcond': '0', + 'msfunc': 'fadd', 'minvfunc': 'fsub', 'mstype': 'integer', + 'minitcond': '0'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + def test_map_ordered_set_aggregate(self): + "Map an ordered-set aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer ORDER BY " + "integer) (sfunc = f1, stype = integer)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', 'kind': 'ordered'} + assert dbmap['schema sd'][ + 'aggregate a1(integer ORDER BY integer)'] == expmap + + def test_map_aggregate_restricted(self): + "Map an aggregate with restricted parallel safety" + stmts = [CREATE_STMT2, "CREATE AGGREGATE a1 (integer) (" + "SFUNC = f1, STYPE = integer, PARALLEL = RESTRICTED)"] + dbmap = self.to_map(stmts) + expmap = {'sfunc': 'f1', 'stype': 'integer', 'parallel': 'restricted'} + assert dbmap['schema sd']['aggregate a1(integer)'] == expmap + + +class AggregateToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input aggregates""" + + def test_create_aggregate_simple(self): + "Create a simple aggregate" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(integer)': { + 'sfunc': 'f1', 'stype': 'integer'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == CREATE_STMT2 + assert fix_indent(sql[2]) == "CREATE AGGREGATE sd.a1(integer) " \ + "(SFUNC = sd.f1, STYPE = integer)" + + def test_create_aggregate_sortop(self): + "Create an aggregate that specifies a sort operator" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(float, float)': { + 'language': 'sql', 'returns': 'float', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(float)': { + 'sfunc': 'f1', 'stype': 'float', 'sortop': 'pg_catalog.>'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[2]) == "CREATE AGGREGATE sd.a1(float) " \ + "(SFUNC = sd.f1, STYPE = float, SORTOP = OPERATOR(pg_catalog.>))" + + def test_create_aggregate_init_final(self): + "Create an aggregate with an INITCOND and a FINALFUNC" + inmap = self.std_map() + inmap['schema sd'].update({'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}}) + inmap['schema sd'].update({'function f2(integer)': { + 'language': 'sql', 'returns': 'double precision', + 'source': "SELECT $1::float", 'volatility': 'immutable'}}) + inmap['schema sd'].update({'aggregate a1(integer)': { + 'sfunc': 'f1', 'stype': 'integer', 'initcond': '-1', + 'finalfunc': 'f2'}}) + sql = self.to_sql(inmap) + funcs = sorted(sql[1:3]) + assert fix_indent(funcs[0]) == CREATE_STMT2 + assert fix_indent(funcs[1]) == "CREATE FUNCTION sd.f2(integer) " \ + "RETURNS double precision LANGUAGE sql IMMUTABLE " \ + "AS $_$SELECT $1::float$_$" + assert fix_indent(sql[3]) == "CREATE AGGREGATE sd.a1(integer) " \ + "(SFUNC = sd.f1, STYPE = integer, FINALFUNC = sd.f2, " \ + "INITCOND = '-1')" + + def test_drop_aggregate(self): + "Drop an existing aggregate" + stmts = [CREATE_STMT2, "CREATE AGGREGATE agg1 (integer) " + "(SFUNC = f1, STYPE = integer)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP AGGREGATE sd.agg1(integer)" + assert sql[1] == "DROP FUNCTION sd.f1(integer, integer)" + + def test_create_moving_aggregate(self): + "Create a moving-aggregate mode function" + inmap = self.std_map() + inmap['schema sd'].update( + {'function fadd(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE4, + 'volatility': 'immutable'}, + 'function fsub(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE5, + 'volatility': 'immutable'}, + 'aggregate a1(integer)': { + 'sfunc': 'fadd', 'stype': 'integer', 'initcond': '0', + 'msfunc': 'fadd', 'minvfunc': 'fsub', 'mstype': 'integer', + 'minitcond': '0'}}) + sql = self.to_sql(inmap, [CREATE_STMT4, CREATE_STMT5]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer) (" \ + "SFUNC = sd.fadd, STYPE = integer, INITCOND = '0', " \ + "MSFUNC = sd.fadd, MINVFUNC = sd.fsub, MSTYPE = integer, " \ + "MINITCOND = '0')" + + def test_create_hypothetical_set_aggregate(self): + "Create a hypothetical-set aggregate" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}, + 'aggregate a1(integer ORDER BY integer)': { + 'kind': 'hypothetical', 'sfunc': 'f1', 'stype': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer " \ + "ORDER BY integer) (SFUNC = sd.f1, STYPE = integer, HYPOTHETICAL)" + + def test_create_aggregate_parallel_safe(self): + "Create an aggregate with parallel safety" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, integer)': { + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'volatility': 'immutable'}, + 'aggregate a1(integer ORDER BY integer)': { + 'sfunc': 'f1', 'stype': 'integer', 'parallel': 'safe'}}) + sql = self.to_sql(inmap, [CREATE_STMT2]) + assert fix_indent(sql[0]) == "CREATE AGGREGATE sd.a1(integer " \ + "ORDER BY integer) (SFUNC = sd.f1, STYPE = integer, " \ + "PARALLEL = SAFE)" diff --git a/tests/dbobject/test_index.py b/tests/dbobject/test_index.py new file mode 100644 index 0000000..54917e1 --- /dev/null +++ b/tests/dbobject/test_index.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +"""Test indexes""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +CREATE_STMT = "CREATE INDEX t1_idx ON t1 (c1)" +COMMENT_STMT = "COMMENT ON INDEX sd.t1_idx IS 'Test index t1_idx'" + + +class IndexToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created indexes""" + + def test_index_1(self): + "Map a single-column index" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_2(self): + "Map a two-column index" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 TEXT)", + "CREATE UNIQUE INDEX t1_idx ON t1 (c1, c2)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1', 'c2'], + 'unique': True}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_3(self): + "Map a table with a unique index and a non-unique GIN index" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 CHAR(5), c3 tsvector)", + "CREATE UNIQUE INDEX t1_idx_1 ON t1 (c1, c2)", + "CREATE INDEX t1_idx_2 ON t1 USING gin (c3)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'character(5)'}}, + {'c3': {'type': 'tsvector'}}], + 'indexes': {'t1_idx_1': {'keys': ['c1', 'c2'], + 'unique': True}, + 't1_idx_2': {'keys': ['c3'], + 'access_method': 'gin'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_partial(self): + "Map a table with a partial index" + dbmap = self.to_map([CREATE_TABLE_STMT, + "CREATE INDEX t1_idx ON t1 (c2) WHERE c1 > 42"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2'], + 'predicate': '(c1 > 42)'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_function_simple(self): + "Map an index using a function -- issue #98" + stmts = [CREATE_TABLE_STMT, "CREATE INDEX t1_idx ON t1 ((lower(c2)))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {'lower(c2)': {'type': 'expression'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_function_complex(self): + "Map indexes using nested functions and complex arguments" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 date)", + "CREATE INDEX t1_idx1 ON t1 (substring(c2 from position(" + "'_begin' in c2)), substring(c2 from position('_end' in " + "c2)))", + "CREATE INDEX t1_idx2 ON t1 (extract(month from c3), " + "extract(day from c3))"] + dbmap = self.to_map(stmts) + expr_type = {'type': 'expression'} + key1 = [{"SUBSTRING(c2 FROM POSITION(('_begin'::text) IN (c2)))": + expr_type}, + {"SUBSTRING(c2 FROM POSITION(('_end'::text) IN (c2)))": + expr_type}] + key2 = [{"EXTRACT(month FROM c3)": expr_type}, + {"EXTRACT(day FROM c3)": expr_type}] + if self.db.version < 150000: + key1 = [{'"substring"(c2, "position"(c2, \'_begin\'' + '::text))': expr_type}, + {'"substring"(c2, "position"(c2, \'_end\'' + '::text))': expr_type}] + key2 = [{"date_part('month'::text, c3)": { + 'type': 'expression'}}, + {"date_part('day'::text, c3)": { + 'type': 'expression'}}] + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'indexes': {'t1_idx1': {'keys': key1}, + 't1_idx2': {'keys': key2}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_col_opts(self): + "Map an index with various column options" + stmts = ["CREATE TABLE t1 (c1 cidr, c2 text)", + "CREATE INDEX t1_idx ON t1 (c1 cidr_ops NULLS FIRST, " + "c2 DESC)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'cidr'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {'c1': {'opclass': 'cidr_ops', 'nulls': 'first'}}, + {'c2': {'order': 'desc'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_index_mixed(self): + "Map indexes using functions, a regular column and expressions" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 text)", + "CREATE INDEX t1_idx ON t1 (btrim(c3, 'x') NULLS FIRST, c1, " + "lower(c2) DESC)", + "CREATE INDEX t1_idx2 ON t1 ((c2 || ', ' || c3), " + "(c3 || ' ' || c2))"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': { + 'keys': [{"btrim(c3, 'x'::text)": { + 'type': 'expression', 'nulls': 'first'}}, + 'c1', {'lower(c2)': { + 'type': 'expression', 'order': 'desc'}}]}, + 't1_idx2': { + 'keys': [{"(((c2 || ', '::text) || c3))": { + 'type': 'expression'}}, + {"(((c3 || ' '::text) || c2))": { + 'type': 'expression'}}]}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_cluster(self): + "Map a table with an index and cluster on it" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT, + "CLUSTER t1 USING t1_idx"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index_comment(self): + "Map an index comment" + dbmap = self.to_map([CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['table t1']['indexes']['t1_idx'][ + 'description'] == 'Test index t1_idx' + + def test_map_multicol_index_with_exprs(self): + "Map a multicol index with expressions" + dbmap = self.to_map(["CREATE TABLE holiday (id serial PRIMARY KEY," + "date date NOT NULL, recurring boolean NOT NULL)", + "CREATE UNIQUE INDEX unique_date ON holiday ((" + "CASE WHEN recurring THEN (0)::double precision " + "ELSE date_part('year'::text, date) END), " + "date_part('month'::text, date), " + "date_part('day'::text, date))"]) + fmt = "(\nCASE\n %s\n %s\nEND)" + assert dbmap['schema sd']['table holiday']['indexes'][ + 'unique_date'] == { + 'keys': [ + {fmt % ("WHEN recurring THEN (0)::double precision", + "ELSE date_part('year'::text, date)"): + {'type': 'expression'}}, + {"date_part('month'::text, date)": {'type': 'expression'}}, + {"date_part('day'::text, date)": {'type': 'expression'}}], + 'unique': True} + + +class IndexToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input indexes""" + + def test_create_table_with_index(self): + "Create new table with a single column index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 (c1)" + + def test_add_index(self): + "Add a two-column unique index to an existing table" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1'], 'unique': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON sd.t1 (c2, c1)"] + + def test_add_index_schema(self): + "Add an index to an existing table in a non-default schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1'], 'unique': True}}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON s1.t1 (c2, c1)"] + + def test_add_index_back_compat(self): + "Add a index to an existing table accepting back-compatible spec" + stmts = ["CREATE TABLE t1 (c1 INTEGER NOT NULL, " + "c2 INTEGER NOT NULL, c3 TEXT)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'integer', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'columns': ['c2', 'c1'], 'unique': True, + 'access_method': 'btree'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE UNIQUE INDEX t1_idx ON sd.t1 (c2, c1)"] + + def test_bad_index(self): + "Fail on creating an index without columns or expression" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'access_method': 'btree'}}}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_create_partial(self): + "Create a partial index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2'], + 'predicate': '(c1 > 42)'}}}}) + sql = self.to_sql(inmap, [CREATE_TABLE_STMT]) + assert fix_indent(sql[0]) == \ + "CREATE INDEX t1_idx ON sd.t1 (c2) WHERE (c1 > 42)" + + def test_create_index_function(self): + "Create an index which uses a function" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{ + 'lower(c2)': {'type': 'expression'}}]}}}}) + sql = self.to_sql(inmap) + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 (lower(c2))" + + def test_create_index_col_opts(self): + "Create table and an index with column options" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'cidr'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [{'c1': { + 'opclass': 'cidr_ops', 'nulls': 'first'}}, 'c2']}}}}) + sql = self.to_sql(inmap) + assert sql[1] == "CREATE INDEX t1_idx ON sd.t1 " \ + "(c1 cidr_ops NULLS FIRST, c2)" + + def test_create_index_mixed(self): + "Create indexes using functions, a regular column and expressions" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text, c3 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': [ + {"btrim(c3, 'x'::text)": {'type': 'expression', + 'nulls': 'first'}}, 'c1', + {'lower(c2)': {'type': 'expression', 'order': 'desc'}}]}, + 't1_idx2': {'keys': [ + {"(((c2 || ', '::text) || c3))": {'type': 'expression'}}, + {"(((c3 || ' '::text) || c2))": { + 'type': 'expression'}}]}}}}) + sql = sorted(self.to_sql(inmap, stmts)) + assert sql[0] == "CREATE INDEX t1_idx ON sd.t1 (" \ + "btrim(c3, 'x'::text) NULLS FIRST, c1, lower(c2) DESC)" + assert sql[1] == "CREATE INDEX t1_idx2 ON sd.t1 (" \ + "(((c2 || ', '::text) || c3)), (((c3 || ' '::text) || c2)))" + + def test_create_table_with_index_clustered(self): + "Create new table clustered on a single column index" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap) + assert sql[2] == "CLUSTER sd.t1 USING t1_idx" + + def test_cluster_table_with_index(self): + "Change a table with an index to cluster on it" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'cluster': True}}}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "CLUSTER sd.t1 USING t1_idx" + + def test_uncluster_table_with_index(self): + "Change a table clustered on an index to remove cluster" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, "CLUSTER t1 USING t1_idx"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET WITHOUT CLUSTER" + + def test_comment_on_index(self): + "Create a comment for an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], + 'description': 'Test index t1_idx'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_index_comment(self): + "Drop the comment on an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON INDEX sd.t1_idx IS NULL"] + + def test_change_index_comment(self): + "Change existing comment on an index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], + 'description': 'Changed index t1_idx'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON INDEX sd.t1_idx IS 'Changed index t1_idx'"] + + def test_change_index_keys(self): + "Change keys of an existing index" + stmts = [CREATE_TABLE_STMT, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['btrim(c1)']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP INDEX sd.t1_idx", + "CREATE INDEX t1_idx ON sd.t1 (btrim(c1))"] + + def test_change_order_index_keys(self): + "Change keys order of an existing index" + stmts = [CREATE_TABLE_STMT, "CREATE INDEX t1_idx ON t1 (c1, c2)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c2', 'c1']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP INDEX sd.t1_idx", + "CREATE INDEX t1_idx ON sd.t1 (c2, c1)"] diff --git a/tests/dbobject/test_matview.py b/tests/dbobject/test_matview.py new file mode 100644 index 0000000..469f300 --- /dev/null +++ b/tests/dbobject/test_matview.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +"""Test materialized views""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE = "CREATE TABLE t1 (c1 INTEGER, c2 TEXT, c3 INTEGER)" +VIEW_STMT = "SELECT c1, c3 * 2 AS mc3 FROM t1" +CREATE_STMT = "CREATE MATERIALIZED VIEW sd.mv1 AS " + VIEW_STMT +COMMENT_STMT = "COMMENT ON MATERIALIZED VIEW sd.mv1 IS 'Test matview mv1'" +VIEW_DEFN = " SELECT t1.c1,\n t1.c3 * 2 AS mc3\n FROM sd.t1;" + + +class MatViewToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created materialized views""" + + def test_map_view_simple(self): + "Map a created materialized view" + stmts = [CREATE_TABLE, CREATE_STMT] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'with_data': True, + 'depends_on': ['table t1']} + assert dbmap['schema sd']['materialized view mv1'] == expmap + + def test_map_view_comment(self): + "Map a materialized view with a comment" + dbmap = self.to_map([CREATE_TABLE, CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['materialized view mv1'][ + 'description'] == 'Test matview mv1' + + def test_map_view_index(self): + "Map a materialized view with an index" + stmts = [CREATE_TABLE, CREATE_STMT, + "CREATE INDEX idx1 ON mv1 (mc3)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'with_data': True, + 'indexes': {'idx1': {'keys': ['mc3']}}, + 'depends_on': ['table t1']} + assert dbmap['schema sd']['materialized view mv1'] == expmap + + +class MatViewToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input materialized views""" + + def test_create_view(self): + "Create a materialized view" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': "SELECT c1, c3 * 2 AS mc3 FROM sd.t1", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, " \ + "c2 text, c3 integer)" + assert fix_indent(sql[1]) == "CREATE MATERIALIZED VIEW sd.mv1 AS " \ + "SELECT c1, c3 * 2 AS mc3 FROM sd.t1" + + def test_bad_view_map(self): + "Error creating a materialized view with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'mv1': {'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_view(self): + "Drop an existing materialized view with table dependencies" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT)", + "CREATE TABLE t2 (c1 INTEGER, c3 TEXT)", + "CREATE MATERIALIZED VIEW mv1 AS SELECT t1.c1, c2, c3 " + "FROM t1 JOIN t2 ON (t1.c1 = t2.c1)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP MATERIALIZED VIEW sd.mv1" + # can't control which table will be dropped first + drt1 = 1 + drt2 = 2 + if 't1' in sql[2]: + drt1 = 2 + drt2 = 1 + assert sql[drt1] == "DROP TABLE sd.t1" + assert sql[drt2] == "DROP TABLE sd.t2" + + def test_view_with_comment(self): + "Create a materialized view with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_STMT, 'description': "Test matview mv1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_view_index(self): + "Create an index on a materialized view" + stmts = [CREATE_TABLE, CREATE_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'materialized view mv1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'mc3': {'type': 'integer'}}], + 'definition': VIEW_DEFN, 'indexes': {'idx1': {'keys': ['mc3']}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["CREATE INDEX idx1 ON sd.mv1 (mc3)"] diff --git a/tests/dbobject/test_operator.py b/tests/dbobject/test_operator.py new file mode 100644 index 0000000..3767c47 --- /dev/null +++ b/tests/dbobject/test_operator.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +"""Test operators""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE OPERATOR sd.+ (PROCEDURE = textcat, LEFTARG = text, " \ + "RIGHTARG = text)" +COMMENT_STMT = "COMMENT ON OPERATOR sd.+(text, text) IS 'Test operator +'" + + +class OperatorToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operators""" + + def test_map_operator1(self): + "Map a simple operator" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'procedure': 'textcat'} + assert dbmap['schema sd']['operator +(text, text)'] == expmap + + def test_map_operator_rightarg(self): + "Map a unitary operator with a right argument" + stmts = ["CREATE OPERATOR + (PROCEDURE = upper, RIGHTARG = text)"] + dbmap = self.to_map(stmts) + expmap = {'procedure': 'pg_catalog.upper'} + assert dbmap['schema sd']['operator +(NONE, text)'] == expmap + + def test_map_operator_commutator(self): + "Map an operator with a commutator" + stmts = ["CREATE OPERATOR && (PROCEDURE = int4pl, LEFTARG = integer, " + "RIGHTARG = integer, COMMUTATOR = OPERATOR(sd.&&))"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['operator &&(integer, integer)'] == \ + {'procedure': 'int4pl', 'commutator': 'sd.&&'} + + def test_map_operator_hash(self): + "Map an operator with HASHES clause" + stmts = ["CREATE OPERATOR + (PROCEDURE = texteq, LEFTARG = text, " + "RIGHTARG = text, HASHES)"] + dbmap = self.to_map(stmts) + expmap = {'procedure': 'texteq', 'hashes': True} + assert dbmap['schema sd']['operator +(text, text)'] == expmap + + def test_map_operator_comment(self): + "Map a operator comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator +(text, text)'][ + 'description'] == 'Test operator +' + + +class OperatorToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + def test_create_operator1(self): + "Create a simple operator" + inmap = self.std_map() + inmap['schema sd'].update({'operator +(text, text)': { + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_operator_rightarg(self): + "Create a unitary operator with a right argument" + inmap = self.std_map() + inmap['schema sd'].update({'operator +(NONE, text)': { + 'procedure': 'upper'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE OPERATOR sd.+ (" \ + "PROCEDURE = upper, RIGHTARG = text)" + + def test_create_operator_commutator(self): + "Create an operator with a commutator" + inmap = self.std_map() + inmap['schema sd'].update({'operator &&(integer, integer)': { + 'procedure': 'int4pl', 'commutator': 'sd.&&'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE OPERATOR sd.&& (" \ + "PROCEDURE = int4pl, LEFTARG = integer, RIGHTARG = integer, " \ + "COMMUTATOR = OPERATOR(sd.&&))" + + def test_create_operator_in_schema(self): + "Create a operator within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator +(text, text)': { + 'procedure': 'textcat'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE OPERATOR s1.+ " \ + "(PROCEDURE = textcat, LEFTARG = text, RIGHTARG = text)" + + def test_drop_operator(self): + "Drop an existing operator" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP OPERATOR sd.+(text, text)"] + + def test_operator_with_comment(self): + "Create a operator with a comment" + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Test operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_operator(self): + "Create a comment for an existing operator" + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Test operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_operator_comment(self): + "Drop a comment on an existing operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'returns': 'integer', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON OPERATOR sd.+(text, text) IS NULL"] + + def test_change_operator_comment(self): + "Change existing comment on a operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({ + 'operator +(text, text)': {'description': 'Changed operator +', + 'procedure': 'textcat'}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON OPERATOR sd.+(text, text) IS " + "'Changed operator +'"] diff --git a/tests/dbobject/test_operclass.py b/tests/dbobject/test_operclass.py new file mode 100644 index 0000000..bda1330 --- /dev/null +++ b/tests/dbobject/test_operclass.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +"""Test operator classes""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TYPE_STMT = """ +CREATE TYPE myint; +CREATE FUNCTION myintin(cstring) RETURNS myint AS 'int4in' LANGUAGE internal; +CREATE FUNCTION myintout(myint) RETURNS cstring AS 'int4out' LANGUAGE internal; +CREATE TYPE myint (INPUT = myintin, OUTPUT = myintout); +CREATE FUNCTION myinteq(myint,myint) RETURNS boolean AS 'int4eq' + LANGUAGE internal; +CREATE FUNCTION myintlt(myint,myint) RETURNS boolean AS 'int4lt' + LANGUAGE internal; +CREATE OPERATOR < (PROCEDURE = myintlt, LEFTARG = myint, RIGHTARG = myint); +CREATE OPERATOR = (PROCEDURE = myinteq, LEFTARG = myint, RIGHTARG = myint); +CREATE FUNCTION btmyintcmp(myint,myint) RETURNS integer AS 'btint4cmp' + LANGUAGE internal; +""" + +CREATE_STMT = "CREATE OPERATOR CLASS oc1 FOR TYPE integer USING btree " \ + "AS OPERATOR 1 <, OPERATOR 3 =, FUNCTION 1 btint4cmp(integer,integer)" +CREATE_STMT_LONG = "CREATE OPERATOR CLASS sd.oc1 FOR TYPE integer " \ + "USING btree AS OPERATOR 1 <(integer,integer), " \ + "OPERATOR 3 =(integer,integer), FUNCTION 1 btint4cmp(integer,integer)" +COMMENT_STMT = "COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS " \ + "'Test operator class oc1'" + + +class OperatorClassToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operator classes""" + + superuser = True + + def test_map_operclass1(self): + "Map an operator class" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['operator class oc1 using btree'] == \ + {'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}} + + def test_map_operclass_default(self): + "Map a default operator class" + stmts = [CREATE_TYPE_STMT, + "CREATE OPERATOR CLASS oc1 DEFAULT FOR TYPE sd.myint " + "USING btree AS OPERATOR 1 <, OPERATOR 3 =, " + "FUNCTION 1 btmyintcmp(sd.myint,sd.myint)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['operator class oc1 using btree'] == \ + {'type': 'myint', 'default': True, + 'operators': {1: 'sd.<(sd.myint,sd.myint)', + 3: 'sd.=(sd.myint,sd.myint)'}, + 'functions': {1: 'sd.btmyintcmp(sd.myint,sd.myint)'}} + + def test_map_operclass_comment(self): + "Map an operator class comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator class oc1 using btree'][ + 'description'] == 'Test operator class oc1' + + +class OperatorClassToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + @pytest.mark.xfail + def test_create_operclass1(self): + "Create an operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_LONG + + def test_create_operclass_default(self): + "Create a default operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'default': True, 'type': 'myint', + 'operators': {1: 'sd.<(sd.myint,sd.myint)', + 3: 'sd.=(sd.myint,sd.myint)'}, + 'functions': {1: 'sd.btmyintcmp(sd.myint,sd.myint)'}}}) + sql = self.to_sql(inmap, [CREATE_TYPE_STMT], superuser=True) + assert fix_indent(sql[0]) == "CREATE OPERATOR CLASS sd.oc1 DEFAULT " \ + "FOR TYPE sd.myint USING btree AS OPERATOR 1 " \ + "sd.<(sd.myint,sd.myint), OPERATOR 3 sd.=(sd.myint,sd.myint), "\ + "FUNCTION 1 sd.btmyintcmp(sd.myint,sd.myint)" + + @pytest.mark.xfail + def test_create_operclass_in_schema(self): + "Create a operator within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE OPERATOR CLASS s1.oc1 FOR " \ + "TYPE integer USING btree AS OPERATOR 1 <(integer,integer), " \ + "OPERATOR 3 =(integer,integer), " \ + "FUNCTION 1 btint4cmp(integer,integer)" + + def test_drop_operclass(self): + "Drop an existing operator" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP OPERATOR CLASS sd.oc1 USING btree", + "DROP OPERATOR FAMILY sd.oc1 USING btree"] + + @pytest.mark.xfail + def test_operclass_with_comment(self): + "Create an operator class with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Test operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_LONG + assert sql[1] == COMMENT_STMT + + def test_comment_on_operclass(self): + "Create a comment for an existing operator class" + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Test operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_operclass_comment(self): + "Drop the existing comment on an operator class" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS NULL"] + + def test_change_operclass_comment(self): + "Change existing comment on an operator class" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator class oc1 using btree': { + 'description': 'Changed operator class oc1', 'type': 'integer', + 'operators': {1: '<(integer,integer)', 3: '=(integer,integer)'}, + 'functions': {1: 'btint4cmp(integer,integer)'}}, + 'operator family oc1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR CLASS sd.oc1 USING btree IS " + "'Changed operator class oc1'"] diff --git a/tests/dbobject/test_operfamily.py b/tests/dbobject/test_operfamily.py new file mode 100644 index 0000000..ea1fcb3 --- /dev/null +++ b/tests/dbobject/test_operfamily.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""Test operator families""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE OPERATOR FAMILY sd.of1 USING btree" +COMMENT_STMT = "COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS " \ + "'Test operator family of1'" + + +class OperatorFamilyToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing operators""" + + superuser = True + + def test_map_operfam(self): + "Map an operator family" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['operator family of1 using btree'] == {} + + def test_map_operfam_comment(self): + "Map an operator family comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['operator family of1 using btree'][ + 'description'] == 'Test operator family of1' + + +class OperatorFamilyToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input operators""" + + def test_create_operfam(self): + "Create an operator family" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': {}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_operfam_in_schema(self): + "Create an operator family within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'operator family of1 using btree': {}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == \ + "CREATE OPERATOR FAMILY s1.of1 USING btree" + + def test_drop_operfam(self): + "Drop an existing operator family" + sql = self.to_sql(self.std_map(), [CREATE_STMT], superuser=True) + assert sql == ["DROP OPERATOR FAMILY sd.of1 USING btree"] + + def test_operfam_with_comment(self): + "Create an operator family with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Test operator family of1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_operfam(self): + "Create a comment for an existing operator family" + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Test operator family of1'}}) + sql = self.to_sql(inmap, [CREATE_STMT], superuser=True) + assert sql == [COMMENT_STMT] + + def test_drop_operfam_comment(self): + "Drop a comment on an existing operator family" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': {}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS NULL"] + + def test_change_operfam_comment(self): + "Change existing comment on a operator" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'operator family of1 using btree': { + 'description': 'Changed operator family of1'}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == ["COMMENT ON OPERATOR FAMILY sd.of1 USING btree IS " + "'Changed operator family of1'"] diff --git a/tests/dbobject/test_owner.py b/tests/dbobject/test_owner.py new file mode 100644 index 0000000..1dbd777 --- /dev/null +++ b/tests/dbobject/test_owner.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +"""Test object ownership + +The majority of other tests exclude owner information. These +explicitly request it. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +SOURCE1 = "SELECT 'dummy'::text" +SOURCE2 = "SELECT $1 * $2" +CREATE_FUNC = "CREATE FUNCTION sd.f1() RETURNS text LANGUAGE sql IMMUTABLE " \ + "AS $_$%s$_$" % SOURCE1 +CREATE_TYPE = "CREATE TYPE sd.t1 AS (x integer, y integer)" + + +class OwnerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of object owner information""" + + def test_map_type(self): + "Map a composite type" + dbmap = self.to_map([CREATE_TYPE], no_owner=False) + expmap = {'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}], + 'owner': self.db.user} + assert dbmap['schema sd']['type t1'] == expmap + + def test_map_table(self): + "Map a table" + dbmap = self.to_map([CREATE_TABLE], no_owner=False) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'owner': self.db.user} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_function(self): + "Map a function" + dbmap = self.to_map([CREATE_FUNC], no_owner=False) + expmap = {'language': 'sql', 'returns': 'text', 'owner': self.db.user, + 'source': SOURCE1, 'volatility': 'immutable'} + assert dbmap['schema sd']['function f1()'] == expmap + + +class OwnerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of owner object information""" + + def test_create_type(self): + "Create a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}], + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TYPE + assert sql[1] == "ALTER TYPE sd.t1 OWNER TO %s" % self.db.user + + def test_create_table(self): + "Create a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE + assert sql[1] == "ALTER TABLE sd.t1 OWNER TO %s" % self.db.user + + def test_create_function(self): + "Create a function in a schema and a comment" + inmap = self.std_map() + inmap.update({'schema s1': {'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'description': 'Test function'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + # skip first two statements as those are tested elsewhere + assert sql[2] == "ALTER FUNCTION s1.f1() OWNER TO %s" % self.db.user + # to verify correct order of invocation of ownable and commentable + assert sql[3] == "COMMENT ON FUNCTION s1.f1() IS 'Test function'" + + def test_create_function_default_args(self): + "Create a function with default arguments" + inmap = self.std_map() + inmap['schema sd'].update({ + 'function f1(integer, INOUT integer)': { + 'allargs': 'integer, INOUT integer DEFAULT 1', + 'language': 'sql', 'returns': 'integer', 'source': SOURCE2, + 'owner': self.db.user}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[1]) == \ + "CREATE FUNCTION sd.f1(integer, INOUT integer DEFAULT 1) " \ + "RETURNS integer LANGUAGE sql AS $_$%s$_$" % SOURCE2 + assert sql[2] == "ALTER FUNCTION sd.f1(integer, INOUT integer) " \ + "OWNER TO %s" % self.db.user + + def test_change_table_owner(self): + "Change the owner of a table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': 'someuser'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert sql[0] == "ALTER TABLE sd.t1 OWNER TO someuser" + + def test_change_table_owner_delim(self): + "Change the owner of a table with delimited identifiers" + inmap = self.std_map() + inmap.update({'schema a-schema': {'table a-table': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': 'someuser'}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA \"a-schema\"", + "CREATE TABLE \"a-schema\".\"a-table\" (" + "c1 integer, c2 text)"]) + assert sql[0] == "ALTER TABLE \"a-schema\".\"a-table\" OWNER TO " \ + "someuser" diff --git a/tests/dbobject/test_privs.py b/tests/dbobject/test_privs.py new file mode 100644 index 0000000..1a230c6 --- /dev/null +++ b/tests/dbobject/test_privs.py @@ -0,0 +1,446 @@ +# -*- coding: utf-8 -*- +"""Test object privileges + +The majority of other tests exclude access privileges. These +explicitly request it. In addition, the roles 'user1' and 'user2' +are created if they don't exist. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_TABLE = "CREATE TABLE t1 (c1 integer, c2 text)" +SOURCE1 = "SELECT 'dummy'::text" +CREATE_FUNC = "CREATE FUNCTION f1() RETURNS text LANGUAGE sql IMMUTABLE AS " \ + "$_$%s$_$" % SOURCE1 +CREATE_FDW = "CREATE FOREIGN DATA WRAPPER fdw1" +CREATE_FS = "CREATE SERVER fs1 FOREIGN DATA WRAPPER fdw1" +GRANT_SELECT = "GRANT SELECT ON TABLE sd.t1 TO %s" +GRANT_INSUPD = "GRANT INSERT, UPDATE ON TABLE sd.t1 TO %s" + + +def check_extra_users(db): + "Check existence of extra test users" + for user in ['user1', 'user2']: + row = db.fetchone("SELECT 1 FROM pg_roles WHERE rolname = %s", (user,)) + if row is None: + db.execute_commit("CREATE ROLE %s" % user) + + +class PrivilegeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of object privilege information""" + + def setUp(self): + super(DatabaseToMapTestCase, self).setUp() + check_extra_users(self.db) + + def test_map_schema(self): + "Map a schema with some GRANTs" + stmts = ["CREATE SCHEMA s1", "GRANT USAGE ON SCHEMA s1 TO PUBLIC", + "GRANT CREATE, USAGE ON SCHEMA s1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['usage']}, + {'user1': ['all']}]}) + assert dbmap['schema s1'] == expmap + + def test_map_table(self): + "Map a table with various GRANTs" + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', GRANT_INSUPD % 'user1', + "GRANT REFERENCES, TRIGGER ON t1 TO user2 WITH GRANT OPTION"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'columns': [ + {'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}, + {'user2': [{'trigger': {'grantable': True}}, + {'references': { + 'grantable': True}}]}]}) + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_column(self): + "Map a table with GRANTs on column" + self.maxDiff = None + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', + "GRANT INSERT (c1, c2) ON t1 TO user1", + "GRANT INSERT (c2), UPDATE (c2) ON t1 TO user2"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'columns': [ + {'c1': {'type': 'integer', 'privileges': [{'user1': ['insert']}]}}, + {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}, {'user2': [ + 'insert', 'update']}]}}], 'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['select']}]}) + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_sequence(self): + "Map a sequence with various GRANTs" + stmts = ["CREATE SEQUENCE seq1", + "GRANT SELECT ON SEQUENCE seq1 TO PUBLIC", + "GRANT USAGE, UPDATE ON SEQUENCE seq1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges({'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, + 'cache_value': 1, 'privileges': [ + {self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['usage', 'update']}]}) + assert dbmap['schema sd']['sequence seq1'] == expmap + + def test_map_view(self): + "Map a view with various GRANTs" + stmts = ["CREATE VIEW v1 AS SELECT now()::date AS today", + "GRANT SELECT ON v1 TO PUBLIC", + "GRANT REFERENCES ON v1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = self.sort_privileges( + {'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT now()::date AS today;", + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['references']}]}) + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_function(self): + "Map a function with a GRANT and REVOKE from PUBLIC" + stmts = [CREATE_FUNC, "REVOKE ALL ON FUNCTION f1() FROM PUBLIC", + "GRANT EXECUTE ON FUNCTION f1() TO user1"] + dbmap = self.to_map(stmts, no_privs=False) + expmap = {'language': 'sql', 'returns': 'text', + 'source': SOURCE1, 'volatility': 'immutable', + 'privileges': [{self.db.user: ['execute']}, + {'user1': ['execute']}]} + assert dbmap['schema sd']['function f1()'] == expmap + + def test_map_fd_wrapper(self): + "Map a foreign data wrapper with a GRANT" + stmts = [CREATE_FDW, + "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = self.sort_privileges({'privileges': + [{self.db.user: ['usage']}, + {'PUBLIC': ['usage']}]}) + assert dbmap['foreign data wrapper fdw1'] == expmap + + def test_map_server(self): + "Map a foreign server with a GRANT" + stmts = [CREATE_FDW, CREATE_FS, + "GRANT USAGE ON FOREIGN SERVER fs1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = {'privileges': [{self.db.user: ['usage']}, + {'user1': ['usage']}]} + assert dbmap['foreign data wrapper fdw1']['server fs1'] == expmap + + def test_map_foreign_table(self): + "Map a foreign table with various GRANTs" + stmts = [CREATE_FDW, CREATE_FS, + "CREATE FOREIGN TABLE ft1 (c1 integer, c2 text) SERVER fs1", + "GRANT SELECT ON ft1 TO PUBLIC", + "GRANT INSERT, UPDATE ON ft1 TO user1"] + dbmap = self.to_map(stmts, no_privs=False, superuser=True) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]} + assert dbmap['schema sd']['foreign table ft1'] == \ + self.sort_privileges(expmap) + + +class PrivilegeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of privilege information (GRANTs)""" + + def setUp(self): + super(InputMapToSqlTestCase, self).setUp() + check_extra_users(self.db) + + def test_create_schema(self): + "Create a schema with various privileges" + inmap = self.std_map() + inmap.update({'schema s1': { + 'owner': self.db.user, 'privileges': [{ + self.db.user: ['all']}, {'PUBLIC': ['usage', 'create']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE SCHEMA + # sql[1] = ALTER SCHEMA OWNER + assert sql[2] == "GRANT ALL ON SCHEMA s1 TO %s" % self.db.user + assert sql[3] == "GRANT ALL ON SCHEMA s1 TO PUBLIC" + + def test_schema_new_grant(self): + "Grant privileges on an existing schema" + inmap = self.std_map() + inmap.update({'schema s1': { + 'owner': self.db.user, 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['create']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE SCHEMA s1"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON SCHEMA s1 TO %s" % self.db.user + assert sql[1] == "GRANT CREATE ON SCHEMA s1 TO PUBLIC" + + def test_create_table(self): + "Create a table with various privileges" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}, + {'user2': [{'trigger': {'grantable': True}}, + {'references': {'grantable': True}}]}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[3] == GRANT_SELECT % 'PUBLIC' + assert sql[4] == GRANT_INSUPD % 'user1' + assert sql[5] == "GRANT TRIGGER, REFERENCES ON TABLE sd.t1 " \ + "TO user2 WITH GRANT OPTION" + + def test_create_column_grants(self): + "Create a table with colum-level privileges" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'privileges': [{'user1': [ + 'insert']}]}}, {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}, {'user2': ['insert', 'update']}]}}], + 'owner': self.db.user, 'privileges': [{self.db.user: ['all']}, + {'PUBLIC': ['select']}]}}) + sql = self.to_sql(inmap) + assert len(sql) == 7 + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[3] == GRANT_SELECT % 'PUBLIC' + assert sql[4] == "GRANT INSERT (c1) ON TABLE sd.t1 TO user1" + assert sql[5] == "GRANT INSERT (c2) ON TABLE sd.t1 TO user1" + assert sql[6] == "GRANT INSERT (c2), UPDATE (c2) ON TABLE sd.t1 " \ + "TO user2" + + def test_table_new_grant(self): + "Grant select privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert len(sql) == 2 + sql = sorted(sql) + assert sql[0] == "GRANT ALL ON TABLE sd.t1 TO %s" % self.db.user + assert sql[1] == GRANT_SELECT % 'user1' + + def test_table_change_grant(self): + "Grant select privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = self.to_sql(inmap, [CREATE_TABLE, GRANT_SELECT % 'user1']) + assert len(sql) == 3 + assert sorted(sql) == [GRANT_INSUPD % 'user1', GRANT_SELECT % 'PUBLIC', + "REVOKE SELECT ON TABLE sd.t1 FROM user1"] + + def test_column_change_grants(self): + "Change existing colum-level privileges" + inmap = self.std_map() + inmap['schema sd'].update( + {'table t1': {'columns': [{'c1': { + 'type': 'integer', 'privileges': [ + {'user1': ['insert']}, {'user2': ['insert', 'update']}]}}, + {'c2': {'type': 'text', 'privileges': [ + {'user1': ['insert']}]}}], + 'owner': self.db.user, 'privileges': [ + {self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', + "GRANT INSERT (c1, c2) ON t1 TO user1", + "GRANT INSERT (c2), UPDATE (c2) ON t1 TO user2"] + sql = self.to_sql(inmap, stmts) + assert len(sql) == 2 + assert sql[0] == "GRANT INSERT (c1), UPDATE (c1) ON TABLE sd.t1 " \ + "TO user2" + assert sql[1] == "REVOKE INSERT (c2), UPDATE (c2) ON TABLE sd.t1 " \ + "FROM user2" + + def test_table_revoke_all(self): + "Revoke all privileges on an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'owner': self.db.user}}) + stmts = [CREATE_TABLE, GRANT_SELECT % 'PUBLIC', GRANT_INSUPD % 'user1'] + sql = sorted(self.to_sql(inmap, stmts)) + assert len(sql) == 3 + assert sql[0] == "REVOKE ALL ON TABLE sd.t1 FROM %s" % self.db.user + assert sql[1] == "REVOKE INSERT, UPDATE ON TABLE sd.t1 FROM user1" + assert sql[2] == "REVOKE SELECT ON TABLE sd.t1 FROM PUBLIC" + + def test_create_sequence(self): + "Create a sequence with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE SEQUENCE + # sql[1] = ALTER SEQUENCE OWNER + assert sql[2] == "GRANT ALL ON SEQUENCE sd.seq1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON SEQUENCE sd.seq1 TO PUBLIC" + + def test_sequence_new_grant(self): + "Grant privileges on an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE SEQUENCE seq1"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON SEQUENCE sd.seq1 TO %s" % self.db.user + assert sql[1] == "GRANT SELECT ON SEQUENCE sd.seq1 TO PUBLIC" + + def test_create_view(self): + "Create a view with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'definition': " SELECT now()::date AS today;", + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE VIEW + # sql[1] = ALTER VIEW OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.v1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON TABLE sd.v1 TO user1" + + def test_view_new_grant(self): + "Grant privileges on an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT now()::date AS today;", + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user1': ['select']}]}}) + sql = sorted(self.to_sql(inmap, ["CREATE VIEW v1 AS " + "SELECT now()::date AS today"])) + assert len(sql) == 2 + assert sql[0] == "GRANT ALL ON TABLE sd.v1 TO %s" % self.db.user + assert sql[1] == "GRANT SELECT ON TABLE sd.v1 TO user1" + + def test_create_function(self): + "Create a function with some privileges" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['execute']}]}}) + sql = self.to_sql(inmap) + # sql[0:1] = SET, CREATE FUNCTION + # sql[2] = ALTER FUNCTION OWNER + assert sql[3] == "GRANT EXECUTE ON FUNCTION sd.f1() TO %s" % ( + self.db.user) + assert sql[4] == "GRANT EXECUTE ON FUNCTION sd.f1() TO PUBLIC" + + def test_function_new_grant(self): + "Grant privileges on an existing function" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'sql', 'returns': 'text', 'source': SOURCE1, + 'volatility': 'immutable', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['execute']}]}}) + sql = self.to_sql(inmap, [CREATE_FUNC]) + assert len(sql) == 2 + sql = sorted(sql) + # assumes self.db.user > PUBLIC + assert sql[0] == "GRANT EXECUTE ON FUNCTION sd.f1() TO PUBLIC" + assert sql[1] == "GRANT EXECUTE ON FUNCTION sd.f1() TO %s" % ( + self.db.user) + + def test_create_fd_wrapper(self): + "Create a foreign data wrapper with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['usage']}]}}) + sql = self.to_sql(inmap) + # sql[0] = CREATE FDW + # sql[1] = ALTER FDW OWNER + assert sql[2] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 " \ + "TO %s" % self.db.user + assert sql[3] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC" + + def test_fd_wrapper_new_grant(self): + "Grant privileges on an existing foreign data wrapper" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['usage']}]}}) + sql = sorted(self.to_sql(inmap, [CREATE_FDW], superuser=True)) + assert len(sql) == 2 + # assumes self.db.user > PUBLIC + assert sql[0] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 TO PUBLIC" + assert sql[1] == "GRANT USAGE ON FOREIGN DATA WRAPPER fdw1 " \ + "TO %s" % self.db.user + + def test_create_server(self): + "Create a foreign server with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user2': ['usage']}]}}}) + sql = self.to_sql(inmap, [CREATE_FDW], superuser=True) + # sql[0] = CREATE SERVER + # sql[1] = ALTER SERVER OWNER + assert sql[2] == "GRANT USAGE ON FOREIGN SERVER fs1 TO %s" % \ + self.db.user + assert sql[3] == "GRANT USAGE ON FOREIGN SERVER fs1 TO user2" + + def test_server_new_grant(self): + "Grant privileges on an existing foreign server" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': { + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'user2': ['usage']}]}}}) + sql = sorted(self.to_sql(inmap, [CREATE_FDW, CREATE_FS], + superuser=True)) + assert len(sql) == 2 + assert sql[0] == "GRANT USAGE ON FOREIGN SERVER fs1 TO %s" % \ + self.db.user + assert sql[1] == "GRANT USAGE ON FOREIGN SERVER fs1 TO user2" + + def test_create_foreign_table(self): + "Create a foreign table with some privileges" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'server': 'fs1', + 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = self.to_sql(inmap, [CREATE_FDW, CREATE_FS], superuser=True) + # sql[0] = CREATE TABLE + # sql[1] = ALTER TABLE OWNER + assert sql[2] == "GRANT ALL ON TABLE sd.ft1 TO %s" % self.db.user + assert sql[3] == "GRANT SELECT ON TABLE sd.ft1 TO PUBLIC" + assert sql[4] == "GRANT INSERT, UPDATE ON TABLE sd.ft1 TO user1" + + def test_foreign_table_new_grant(self): + "Grant privileges on an existing foreign table" + inmap = self.std_map() + inmap.update({'foreign data wrapper fdw1': {'server fs1': {}}}) + inmap['schema sd'].update({'foreign table ft1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'server': 'fs1', 'owner': self.db.user, + 'privileges': [{self.db.user: ['all']}, {'PUBLIC': ['select']}, + {'user1': ['insert', 'update']}]}}) + sql = sorted(self.to_sql(inmap, [ + CREATE_FDW, CREATE_FS, + "CREATE FOREIGN TABLE ft1 (c1 integer, c2 text) SERVER fs1"], + superuser=True)) + assert len(sql) == 3 + assert sql[0] == "GRANT ALL ON TABLE sd.ft1 TO %s" % self.db.user + assert sql[1] == "GRANT INSERT, UPDATE ON TABLE sd.ft1 TO user1" + assert sql[2] == "GRANT SELECT ON TABLE sd.ft1 TO PUBLIC" diff --git a/tests/dbobject/test_rule.py b/tests/dbobject/test_rule.py new file mode 100644 index 0000000..90e0304 --- /dev/null +++ b/tests/dbobject/test_rule.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +"""Test rules""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TABLE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +CREATE_STMT = "CREATE RULE r1 AS ON %s TO sd.t1 DO %s" +COMMENT_STMT = "COMMENT ON RULE r1 ON sd.t1 IS 'Test rule r1'" + + +class RuleToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing rules""" + + def test_map_rule_nothing(self): + "Map a rule to do nothing" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules'] == \ + {'r1': {'event': 'insert', 'actions': 'NOTHING'}} + + def test_map_rule_instead(self): + "Map rule with an INSTEAD action" + stmts = [CREATE_TABLE_STMT, + CREATE_STMT % ('UPDATE', 'INSTEAD NOTHING')] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules'] == \ + {'r1': {'event': 'update', 'instead': True, 'actions': 'NOTHING'}} + + def test_map_rule_conditional(self): + "Map rule with a qualification" + stmts = [CREATE_TABLE_STMT, + "CREATE RULE r1 AS ON DELETE TO t1 WHERE OLD.c1 < 1000 " + "DO INSERT INTO t1 VALUES (OLD.c1 + 1000, OLD.c2)"] + dbmap = self.to_map(stmts) + fmt = " %s\n %s" + action = fmt % ("INSERT INTO sd.t1 (c1, c2)", + " VALUES ((old.c1 + 1000), old.c2)") + expmap = {'r1': {'event': 'delete', 'condition': "(old.c1 < 1000)", + 'actions': action}} + assert dbmap['schema sd']['table t1']['rules'] == expmap + + def test_map_rule_multi_actions(self): + "Map rule with multiple actions" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ( + 'UPDATE', "(INSERT INTO t1 (c1) VALUES (old.c1 + 100); " + "INSERT INTO t1 (c1) VALUES (old.c1 + 200))")] + dbmap = self.to_map(stmts) + ins = "INSERT INTO sd.t1 (c1)" + fmt = "( %s\n %s\n %s\n %s\n)" + actions = fmt % (ins, " VALUES ((old.c1 + 100));", + ins, " VALUES ((old.c1 + 200));") + expmap = {'r1': {'event': 'update', 'actions': actions}} + assert dbmap['schema sd']['table t1']['rules'] == expmap + + def test_map_rule_comment(self): + "Map a rule comment" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['rules']['r1'][ + 'description'] == 'Test rule r1' + + +class RuleToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input rules""" + + def test_create_rule_nothing(self): + "Create a rule" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == CREATE_STMT % ('INSERT', 'NOTHING') + + def test_create_rule_instead(self): + "Create a rule with an INSTEAD action" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'update', 'instead': True, + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON UPDATE TO sd.t1 " \ + "DO INSTEAD NOTHING" + + def test_create_rule_conditional(self): + "Create a rule with qualification" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'delete', + 'condition': "old.c1 < 1000", + 'actions': "INSERT INTO t1 VALUES (" + "old.c1 + 1000, old.c2)"}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON DELETE TO sd.t1 " \ + "WHERE old.c1 < 1000 DO INSERT INTO t1 VALUES " \ + "(old.c1 + 1000, old.c2)" + + def test_create_rule_multi_actions(self): + "Create a rule with multiple actions" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'update', 'actions': + "(INSERT INTO t1 VALUES (old.c1 + 100); " + "INSERT INTO t1 VALUES (old.c1 + 200));)"}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON UPDATE " \ + "TO sd.t1 DO (INSERT INTO t1 VALUES (old.c1 + 100); " \ + "INSERT INTO t1 VALUES (old.c1 + 200));)" + + def test_create_rule_in_schema(self): + "Create a rule within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': { + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', + 'actions': 'NOTHING'}}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[1]) == "CREATE RULE r1 AS ON INSERT TO s1.t1 " \ + "DO NOTHING" + + def test_drop_rule(self): + "Drop an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP RULE r1 ON sd.t1"] + + def test_drop_rule_table(self): + "Drop an existing rule and the related table" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP RULE r1 ON sd.t1" + assert sql[1] == "DROP TABLE sd.t1" + + def test_rule_with_comment(self): + "Create a rule with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'description': 'Test rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap) + assert sql[2] == COMMENT_STMT + + def test_comment_on_rule(self): + "Create a comment on an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING')] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'description': 'Test rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_rule_comment(self): + "Drop the comment on an existing rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON RULE r1 ON sd.t1 IS NULL"] + + def test_change_rule_comment(self): + "Change existing comment on a rule" + stmts = [CREATE_TABLE_STMT, CREATE_STMT % ('INSERT', 'NOTHING'), + COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'rules': {'r1': {'event': 'insert', + 'description': 'Changed rule r1', + 'actions': 'NOTHING'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON RULE r1 ON sd.t1 IS 'Changed rule r1'"] diff --git a/tests/dbobject/test_schema.py b/tests/dbobject/test_schema.py new file mode 100644 index 0000000..cb6a9a1 --- /dev/null +++ b/tests/dbobject/test_schema.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +"""Test schemas""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_STMT = "CREATE SCHEMA s1" +COMMENT_STMT = "COMMENT ON SCHEMA s1 IS 'Test schema s1'" + + +class SchemaToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created schemas""" + + def test_map_schema(self): + "Map a created schema" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema s1'] == {} + + def test_map_schema_comment(self): + "Map a schema comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema s1'] == {'description': 'Test schema s1'} + + def test_map_select_schema(self): + "Map a single schema when three schemas exist" + stmts = [CREATE_STMT, "CREATE SCHEMA s2", "CREATE SCHEMA s3"] + dbmap = self.to_map(stmts, schemas=['s2']) + assert 'schema s1' not in dbmap + assert dbmap['schema s2'] == {} + assert 'schema s3' not in dbmap + + +class SchemaToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input schemas""" + + def base_schmap(self): + return {'schema s1': {'description': 'Test schema s1'}} + + def test_create_schema(self): + "Create a new schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT] + + def test_bad_schema_map(self): + "Error creating a schema with a bad map" + with pytest.raises(KeyError): + self.to_sql({'s1': {}}) + + def test_drop_schema(self): + "Drop an existing schema" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP SCHEMA s1"] + + def test_rename_schema(self): + "Rename an existing schema" + inmap = self.std_map() + inmap.update({'schema s2': {'oldname': 's1'}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER SCHEMA s1 RENAME TO s2"] + + def test_bad_rename_schema(self): + "Error renaming a non-existing schema" + inmap = self.std_map() + inmap.update({'schema s2': {'oldname': 's3'}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_schema_with_comment(self): + "Create a schema with a comment" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap) + assert sql == [CREATE_STMT, COMMENT_STMT] + + def test_comment_on_schema(self): + "Create a comment for an existing schema" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_schema_comment(self): + "Drop a comment on an existing schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SCHEMA s1 IS NULL"] + + def test_change_schema_comment(self): + "Change existing comment on a schema" + inmap = self.std_map() + inmap.update({'schema s1': {'description': 'Changed schema s1'}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SCHEMA s1 IS 'Changed schema s1'"] + + +class SchemaUndoSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation to revert schemas""" + + def base_schmap(self): + return {'schema s1': {'description': 'Test schema s1'}} + + def test_undo_create_schema(self): + "Revert a schema creation" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + sql = self.to_sql(inmap, revert=True) + assert sql == ["DROP SCHEMA s1"] + + def test_undo_drop_schema(self): + "Revert dropping a schema" + sql = self.to_sql(self.std_map(), [CREATE_STMT], revert=True) + assert sql[0] == CREATE_STMT + + def test_undo_comment_on_schema(self): + "Revert creating comment on a schema" + inmap = self.std_map() + inmap.update(self.base_schmap()) + sql = self.to_sql(inmap, [CREATE_STMT], revert=True) + assert sql == ["COMMENT ON SCHEMA s1 IS NULL"] + + def test_undo_drop_schema_comment(self): + "Revert dropping comment on a schema" + inmap = self.std_map() + inmap.update({'schema s1': {}}) + stmts = [CREATE_STMT, COMMENT_STMT] + sql = self.to_sql(inmap, stmts, revert=True) + assert sql == [COMMENT_STMT] diff --git a/tests/dbobject/test_sequence.py b/tests/dbobject/test_sequence.py new file mode 100644 index 0000000..4353511 --- /dev/null +++ b/tests/dbobject/test_sequence.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +"""Test sequences""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE SEQUENCE seq1" +CREATE_STMT_FULL = "CREATE SEQUENCE sd.seq1 %sSTART WITH 1 INCREMENT BY 1 " \ + "NO MINVALUE NO MAXVALUE CACHE 1" +COMMENT_STMT = "COMMENT ON SEQUENCE sd.seq1 IS 'Test sequence seq1'" + + +class SequenceToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created sequences""" + + def test_map_sequence_simple(self): + "Map a created sequence" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1} + assert dbmap['schema sd']['sequence seq1'] == expmap + + def test_map_sequence_comment(self): + "Map a sequence with a comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['sequence seq1']['description'] == \ + 'Test sequence seq1' + + +class SequenceToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input sequences""" + + def test_create_sequence_simple(self): + "Create a sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, 'data_type': 'integer'}}) + sql = self.to_sql(inmap) + mod = 'AS integer ' + assert fix_indent(sql[0]) == CREATE_STMT_FULL % mod + + def test_create_sequence_in_schema(self): + "Create a sequence within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE SEQUENCE s1.seq1 START WITH 1 " \ + "INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" + + def test_bad_sequence_map(self): + "Error creating a sequence with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_sequence(self): + "Drop an existing sequence" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP SEQUENCE sd.seq1"] + + def test_no_drop_owned_sequence(self): + "Don't drop a sequence owned by a table column" + sql = self.to_sql(self.std_map(), + ["CREATE TABLE t1 (c1 SERIAL, c2 text)"]) + assert sql == ["DROP TABLE sd.t1"] + + def test_rename_sequence(self): + "Rename an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq2': { + 'oldname': 'seq1', 'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER SEQUENCE sd.seq1 RENAME TO seq2"] + + def test_bad_rename_sequence(self): + "Error renaming a non-existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq2': { + 'oldname': 'seq3', 'start_value': 1, 'increment_by': 1, + 'max_value': None, 'min_value': None, 'cache_value': 1}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_change_sequence(self): + "Change sequence attributes" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 5, 'increment_by': 10, 'max_value': None, + 'min_value': None, 'cache_value': 30}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "ALTER SEQUENCE sd.seq1 START WITH 5 " \ + "INCREMENT BY 10 CACHE 30" + + def test_sequence_with_comment(self): + "Create a sequence with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Test sequence seq1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT_FULL % '' + assert sql[1] == COMMENT_STMT + + def test_comment_on_sequence(self): + "Create a comment for an existing sequence" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Test sequence seq1"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_sequence_comment(self): + "Drop the comment on an existing sequence" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON SEQUENCE sd.seq1 IS NULL"] + + def test_change_sequence_comment(self): + "Change existing comment on a sequence" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1, + 'description': "Changed sequence seq1"}}) + sql = self.to_sql(inmap, stmts) + assert sql, ["COMMENT ON SEQUENCE sd.seq1 IS 'Changed sequence seq1'"] + + +class SequenceUndoSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation to revert sequences""" + + def test_undo_create_sequence(self): + "Revert a sequence creation" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}) + sql = self.to_sql(inmap, revert=True) + assert sql == ["DROP SEQUENCE sd.seq1"] + + def test_undo_create_sequence_in_schema(self): + "Revert creating a sequence in a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'sequence seq1': { + 'start_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'cache_value': 1}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"], revert=True) + assert sql == ["DROP SEQUENCE s1.seq1"] + + def test_undo_drop_sequence(self): + "Revert dropping a sequence" + sql = self.to_sql(self.std_map(), [CREATE_STMT], revert=True) + assert fix_indent(sql[0]) == CREATE_STMT_FULL % '' + + def test_undo_change_sequence(self): + "Revert changing sequence attributes" + inmap = self.std_map() + inmap['schema sd'].update({'sequence seq1': { + 'start_value': 5, 'increment_by': 10, 'max_value': None, + 'min_value': None, 'cache_value': 30}}) + sql = self.to_sql(inmap, [CREATE_STMT], revert=True) + assert fix_indent(sql[0]) == "ALTER SEQUENCE sd.seq1 START WITH 1 " \ + "INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1" diff --git a/tests/dbobject/test_static.py b/tests/dbobject/test_static.py new file mode 100644 index 0000000..009fc2d --- /dev/null +++ b/tests/dbobject/test_static.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +"""Test loading of data from and into static tables""" +import os + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase + +CREATE_STMT = "CREATE TABLE t1 (c1 integer, c2 text)" +FILE_PATH = 'table.t1.data' +TABLE_DATA = [(1, 'abc'), (2, 'def'), (3, 'ghi')] +TABLE_DATA2 = [(1, 'abc', 'row 1'), (3, 'ghi', 'row 2'), (2, 'def', 'row 3'), + (3, 'def', 'row 4')] + + +class StaticTableToMapTestCase(DatabaseToMapTestCase): + """Test mapping and copying out of created tables""" + + def tearDown(self): + self.remove_tempfiles() + + def test_copy_static_table(self): + "Copy a two-column table to a file" + self.db.execute(CREATE_STMT) + for row in TABLE_DATA: + self.db.execute("INSERT INTO t1 VALUES (%s, %s)", row) + cfg = {'datacopy': {'schema sd': ['t1']}} + dbmap = self.to_map([], config=cfg) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]} + recs = [] + with open(os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH)) as f: + for line in f: + (c1, c2) = line.split(',') + recs.append((int(c1), c2.rstrip())) + assert recs == TABLE_DATA + + def test_copy_static_table_pk(self): + "Copy a table that has a primary key" + self.db.execute("CREATE TABLE t1 (c1 integer, c2 char(3), c3 text," + "PRIMARY KEY (c2, c1))") + for row in TABLE_DATA2: + self.db.execute("INSERT INTO t1 VALUES (%s, %s, %s)", row) + cfg = {'datacopy': {'schema sd': ['t1']}} + dbmap = self.to_map([], config=cfg) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'character(3)', 'not_null': True}}, + {'c3': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c2', 'c1']}}} + recs = [] + with open(os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH)) as f: + for line in f: + (c1, c2, c3) = line.split(',') + recs.append((int(c1), c2, c3.rstrip())) + assert recs == sorted(TABLE_DATA2) + + +class StaticTableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of data import statements""" + + def test_load_static_table(self): + "Truncate and import a two-column table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + cfg = {'datacopy': {'schema sd': ['t1']}} + sql = self.to_sql(inmap, [CREATE_STMT], config=cfg) + copy_stmt = ("\\copy ", 'sd.t1', " from '", + os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH), "' csv") + assert sql[0] == "TRUNCATE ONLY sd.t1" + assert sql[1] == copy_stmt + + def test_load_static_table_fk(self): + "Truncate and import a table which has a foreign key dependency" + stmts = ["CREATE TABLE t1 (pc1 integer PRIMARY KEY, pc2 text)", + "CREATE TABLE t2 (c1 integer, c2 integer REFERENCES t1, " + "c3 text)"] + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'pc1': {'type': 'integer', 'not_null': True}}, + {'pc2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['pc1']}}}, 'table t2': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'foreign_keys': {'t2_c2_fkey': { + 'columns': ['c2'], + 'references': {'schema': 'sd', 'table': 't1', + 'columns': ['pc1']}}}}}) + cfg = {'datacopy': {'schema sd': ['t1']}} + sql = self.to_sql(inmap, stmts, config=cfg) + copy_stmt = ("\\copy ", 'sd.t1', " from '", + os.path.join(self.cfg['files']['data_path'], + "schema.sd", FILE_PATH), "' csv") + assert sql[0] == "ALTER TABLE sd.t2 DROP CONSTRAINT t2_c2_fkey" + assert sql[1] == "TRUNCATE ONLY sd.t1" + assert sql[2] == copy_stmt + assert sql[3] == "ALTER TABLE sd.t2 ADD CONSTRAINT t2_c2_fkey " \ + "FOREIGN KEY (c2) REFERENCES sd.t1 (pc1)" diff --git a/tests/dbobject/test_table.py b/tests/dbobject/test_table.py new file mode 100644 index 0000000..7aa7edc --- /dev/null +++ b/tests/dbobject/test_table.py @@ -0,0 +1,436 @@ +# -*- coding: utf-8 -*- +"""Test tables""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text)" +COMMENT_STMT = "COMMENT ON TABLE sd.t1 IS 'Test table t1'" +CREATE_STOR_PARAMS = CREATE_STMT + \ + " WITH (fillfactor=90, autovacuum_enabled=false)" + + +class TableToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created tables""" + + def test_map_table_simple(self): + "Map a table with two columns" + dbmap = self.to_map([CREATE_STMT]) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}]} + + def test_map_table_comment(self): + "Map a table comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'description': 'Test table t1'} + + def test_map_table_comment_quotes(self): + "Map a table comment with quotes" + stmts = [CREATE_STMT, "COMMENT ON TABLE t1 IS " + "'A \"special\" person''s table t1'"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'description': "A \"special\" person's table t1"} + + def test_map_column_comments(self): + "Map two column comments" + stmts = [CREATE_STMT, + "COMMENT ON COLUMN t1.c1 IS 'Test column c1 of t1'", + "COMMENT ON COLUMN t1.c2 IS 'Test column c2 of t1'"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1'] == { + 'columns': [{'c1': {'type': 'integer', + 'description': 'Test column c1 of t1'}}, + {'c2': {'type': 'text', + 'description': 'Test column c2 of t1'}}]} + + def test_map_table_options(self): + "Map a table with options" + dbmap = self.to_map([CREATE_STOR_PARAMS]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=90", 'autovacuum_enabled=false']} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_inherit(self): + "Map a table that inherits from two other tables" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c3 integer)", + "CREATE TABLE t3 (c4 text) INHERITS (t1, t2)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'integer', 'inherited': True}}, + {'c4': {'type': 'text'}}], + 'inherits': ['t1', 't2']} + assert dbmap['schema sd']['table t3'] == expmap + + def test_map_inherit_delim(self): + "Map a table that inherits from two other tables (delim identifiers)" + stmts = [CREATE_STMT, "CREATE TABLE \"t-2\" (c3 integer)", + "CREATE TABLE \"t-3\" (c4 text) INHERITS (t1, \"t-2\")"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'integer', 'inherited': True}}, + {'c4': {'type': 'text'}}], + "inherits": ["t1", "t-2"]} + assert dbmap["schema sd"]["table t-3"] == expmap + + def test_map_unlogged_table(self): + "Map an unlogged table" + dbmap = self.to_map(["CREATE UNLOGGED TABLE t1 (c1 integer, c2 text)"]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'unlogged': True} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_table_within_schema(self): + "Map a schema and a table within it" + stmts = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 INTEGER, c2 TEXT)"] + dbmap = self.to_map(stmts) + assert dbmap['schema s1'] == { + 'table t1': {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}} + + def test_map_table_quoted(self): + "Map a schema and a table both of which need to be quoted" + stmts = ['CREATE SCHEMA "a schema"', + 'CREATE TABLE "a schema"."The.Table" ("column" SERIAL, ' + 'c2 TEXT)'] + dbmap = self.to_map(stmts) + assert dbmap['schema a schema']['table The.Table'] == { + 'columns': [{'column': { + 'type': 'integer', 'not_null': True, + 'default': + 'nextval(\'"a schema"."The.Table_column_seq"\'::regclass)'}}, + {'c2': {'type': 'text'}}]} + + def test_map_select_tables(self): + "Map two tables out of three present" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE TABLE t3 (c1 integer, c2 text)"] + dbmap = self.to_map(stmts, tables=['t2', 't1']) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + assert dbmap['schema sd']['table t1'] == expmap + assert dbmap['schema sd']['table t2'] == expmap + assert 'table t3' not in dbmap['schema sd'] + + def test_map_table_sequence(self): + "Map sequence if owned by a table" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c1 integer, c2 text)", + "CREATE SEQUENCE seq1", "ALTER SEQUENCE seq1 OWNED BY t2.c1", + "CREATE SEQUENCE seq2"] + dbmap = self.to_map(stmts, tables=['t2']) + self.db.execute_commit("DROP SEQUENCE seq1") + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]} + assert 'table t1' not in dbmap['schema sd'] + assert dbmap['schema sd']['table t2'] == expmap + assert 'sequence seq1' in dbmap['schema sd'] + assert 'sequence seq2' not in dbmap['schema sd'] + + def test_map_partition_range(self): + "Map a partitioned table with two partitions by range" + spec1 = "FROM ('2015-01-01', MINVALUE) TO ('2016-12-31', 5)" + stmts = ["CREATE TABLE t1 (c1 date, c2 integer, c3 text) " + "PARTITION BY RANGE (c1, c2)", + "CREATE TABLE t1a PARTITION OF t1 FOR VALUES %s" % spec1, + "CREATE TABLE t1b PARTITION OF t1 FOR VALUES " + "FROM ('2017-01-01', 11) TO ('2020-12-31', 15)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'date'}}, + {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'partition_by': {'range': ['c1', 'c2']}} + assert dbmap['schema sd']['table t1'] == expmap + expmap2 = {'partition_bound_spec': spec1, 'partition_of': 't1'} + assert dbmap['schema sd']['table t1a'] == expmap2 + + def test_map_partition_list(self): + "Map a partitioned table with two partitions by list" + spec1 = "IN (1, 3, 5, 7)" + stmts = ["CREATE TABLE t1 (c1 integer, c2 text) " + "PARTITION BY LIST (c1)", + "CREATE TABLE t1a PARTITION OF t1 FOR VALUES %s" % spec1, + "CREATE TABLE t1b PARTITION OF t1 FOR VALUES IN (2, 4, 6)"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'partition_by': {'list': ['c1']}} + assert dbmap['schema sd']['table t1'] == expmap + expmap2 = {'partition_bound_spec': spec1, 'partition_of': 't1'} + assert dbmap['schema sd']['table t1a'] == expmap2 + + +class TableToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table statements from input schemas""" + + def test_create_table_simple(self): + "Create a two-column table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_table_quoted_idents(self): + "Create a table needing quoted identifiers" + inmap = self.std_map() + inmap['schema sd'].update({'table order': { + 'columns': [{'primary': {'type': 'integer'}}, + {'two words': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, quote_reserved=True) + assert fix_indent(sql[0]) == 'CREATE TABLE sd."order" (' \ + '"primary" integer, "two words" text)' + + def test_bad_table_map(self): + "Error creating a table with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_missing_columns(self): + "Error creating a table with no columns" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': {'columns': []}}) + with pytest.raises(ValueError): + self.to_sql(inmap) + + def test_drop_table(self): + "Drop an existing table" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP TABLE sd.t1"] + + def test_rename_table(self): + "Rename an existing table" + inmap = self.std_map() + inmap['schema sd'].update({'table t2': { + 'oldname': 't1', + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER TABLE sd.t1 RENAME TO t2"] + + def test_create_table_options(self): + "Create a table with options" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=90", "autovacuum_enabled=false"]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STOR_PARAMS + + def test_change_table_options(self): + "Change a table's storage parameters" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'options': ["fillfactor=70"]}}) + sql = self.to_sql(inmap, [CREATE_STOR_PARAMS]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET (fillfactor=70)," \ + " RESET (autovacuum_enabled)" + + def test_create_table_within_schema(self): + "Create a new schema and a table within it" + inmap = self.std_map() + inmap.update({'schema s1': {'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + sql = self.to_sql(inmap) + expsql = ["CREATE SCHEMA s1", + "CREATE TABLE s1.t1 (c1 integer, c2 text)"] + for i in range(len(expsql)): + assert fix_indent(sql[i]) == expsql[i] + + def test_unlogged_table(self): + "Create an unlogged table" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], 'unlogged': True}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == \ + "CREATE UNLOGGED TABLE sd.t1 (c1 integer, c2 text)" + + def test_table_owned_by_sequence(self): + "Alter a table to be owned by a table column" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}, + 'sequence seq1': { + 'cache_value': 1, 'increment_by': 1, 'max_value': None, + 'min_value': None, 'start_value': 1, + 'owner_table': 't1', 'owner_column': 'c1'}}) + sql = self.to_sql(inmap, [CREATE_STMT, "CREATE SEQUENCE seq1"]) + assert sql[0] == "ALTER SEQUENCE sd.seq1 OWNED BY sd.t1.c1" + + def test_create_partitioned_tables(self): + "Create a partitioned table and two partitions" + inmap = self.std_map() + spec1 = "FROM ('2015-01-01', MINVALUE) TO ('2016-12-31', 5)" + spec2 = "FROM ('2017-01-01', 11) TO ('2020-12-31', 15)" + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'date'}}, {'c2': {'type': 'integer'}}, + {'c3': {'type': 'text'}}], + 'partition_by': {'range': ['c1', 'c2']}}, 'table t1a': { + 'partition_bound_spec': spec1, 'partition_of': 't1'}, + 'table t1b': {'partition_bound_spec': spec2, + 'partition_of': 't1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 date, " \ + "c2 integer, c3 text) PARTITION BY RANGE (c1, c2)" + assert fix_indent(sql[1]) == ( + "CREATE TABLE sd.t1a PARTITION OF t1 FOR VALUES %s" % spec1) + assert fix_indent(sql[2]) == ( + "CREATE TABLE sd.t1b PARTITION OF t1 FOR VALUES %s" % spec2) + + +class TableCommentToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table and column COMMENT statements""" + + def _tblmap(self): + "Return a table input map with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'description': 'Test table t1', + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + return inmap + + def test_table_with_comment(self): + "Create a table with a comment" + sql = self.to_sql(self._tblmap()) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_table(self): + "Create a comment for an existing table" + sql = self.to_sql(self._tblmap(), [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_table_comment_quotes(self): + "Create a table comment with quotes" + inmap = self._tblmap() + inmap['schema sd']['table t1']['description'] = \ + "A \"special\" person's table t1" + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS " + "'A \"special\" person''s table t1'"] + + def test_drop_table_comment(self): + "Drop a comment on an existing table" + inmap = self._tblmap() + del inmap['schema sd']['table t1']['description'] + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS NULL"] + + def test_change_table_comment(self): + "Change existing comment on a table" + inmap = self._tblmap() + inmap['schema sd']['table t1'].update( + {'description': 'Changed table t1'}) + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql == ["COMMENT ON TABLE sd.t1 IS 'Changed table t1'"] + + def test_create_column_comments(self): + "Create a table with column comments" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Test column c1') + inmap['schema sd']['table t1']['columns'][1]['c2'].update( + description='Test column c2') + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + assert sql[2] == "COMMENT ON COLUMN sd.t1.c1 IS 'Test column c1'" + assert sql[3] == "COMMENT ON COLUMN sd.t1.c2 IS 'Test column c2'" + + def test_add_column_comment(self): + "Add a column comment to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Test column c1') + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS 'Test column c1'" + + def test_add_column_with_comment(self): + "Add a commented column to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'].append({'c3': { + 'description': 'Test column c3', 'type': 'integer'}}) + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 ADD COLUMN c3 integer" + assert sql[1] == "COMMENT ON COLUMN sd.t1.c3 IS 'Test column c3'" + + def test_drop_column_comment(self): + "Drop a column comment on an existing table" + stmts = [CREATE_STMT, COMMENT_STMT, + "COMMENT ON COLUMN t1.c1 IS 'Test column c1'"] + sql = self.to_sql(self._tblmap(), stmts) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS NULL" + + def test_change_column_comment(self): + "Add a column comment to an existing table" + inmap = self._tblmap() + inmap['schema sd']['table t1']['columns'][0]['c1'].update( + description='Changed column c1') + sql = self.to_sql(inmap, [CREATE_STMT, COMMENT_STMT]) + assert sql[0] == "COMMENT ON COLUMN sd.t1.c1 IS 'Changed column c1'" + + +class TableInheritToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table inheritance statements""" + + def test_table_inheritance(self): + "Create a table that inherits from another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}) + inmap['schema sd'].update({'table t2': { + 'columns': [{'c1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'numeric'}}], 'inherits': ['t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert fix_indent(sql[1]) == "CREATE TABLE sd.t2 (c3 numeric) " \ + "INHERITS (sd.t1)" + + def test_table_inherit_delim(self): + "Create a table that inherits from another (delimited identifiers)" + inmap = self.std_map() + inmap.update({'schema s-d': {'table t-1': { + 'columns': [{'c-1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}]}}}) + inmap['schema s-d'].update({'table t-2': { + 'columns': [{'c-1': {'type': 'integer', 'inherited': True}}, + {'c2': {'type': 'text', 'inherited': True}}, + {'c3': {'type': 'numeric'}}], 'inherits': ['t-1']}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA \"s-d\""]) + print(sql) + assert fix_indent(sql[0]) == "CREATE TABLE \"s-d\".\"t-1\" " \ + "(\"c-1\" integer, c2 text)" + assert fix_indent(sql[1]) == "CREATE TABLE \"s-d\".\"t-2\" " \ + "(c3 numeric) INHERITS (\"s-d\".\"t-1\")" + + def test_drop_inherited(self): + "Drop tables that inherit from others" + stmts = [CREATE_STMT, "CREATE TABLE t2 (c3 numeric) INHERITS (t1)", + "CREATE TABLE t3 (c4 date) INHERITS (t2)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql == ["DROP TABLE sd.t3", "DROP TABLE sd.t2", + "DROP TABLE sd.t1"] diff --git a/tests/dbobject/test_tablespace.py b/tests/dbobject/test_tablespace.py new file mode 100644 index 0000000..45cbc7c --- /dev/null +++ b/tests/dbobject/test_tablespace.py @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +"""Test tablespaces + +These tests require the existence of tablespaces ts1 and ts2. +They should be owned by the user running the tests or the user should +have been granted CREATE (or ALL) privileges on the tablespaces. +""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + + +CREATE_TABLE = "CREATE TABLE sd.t1 (c1 integer, c2 text) TABLESPACE ts1" + + +class ToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created tables""" + + def test_map_table(self): + "Map a table using a tablespace" + dbmap = self.to_map([CREATE_TABLE]) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'tablespace': 'ts1'} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_primary_key(self): + "Map a table with a PRIMARY KEY using a tablespace" + dbmap = self.to_map(["CREATE TABLE t1 (c1 integer PRIMARY KEY " + "USING INDEX TABLESPACE ts1, c2 text)"]) + expmap = {'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], + 'tablespace': 'ts1'}}} + assert dbmap['schema sd']['table t1'] == expmap + + def test_map_index(self): + "Map an index using a tablespace" + dbmap = self.to_map(["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE UNIQUE INDEX t1_idx ON t1 (c1) " + "TABLESPACE ts1"]) + expmap = {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts1', + 'unique': True}} + assert dbmap['schema sd']['table t1']['indexes'] == expmap + + +class ToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation of table statements from input schemas""" + + def test_create_table(self): + "Create a table in a tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE + + def test_move_table(self): + "Move a table from one tablespace to another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'tablespace': 'ts2'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert fix_indent(sql[0]) == "ALTER TABLE sd.t1 SET TABLESPACE ts2" + + def test_create_primary_key(self): + "Create a table with a PRIMARY KEY in a different tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer', 'not_null': True}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1'], + 'tablespace': 'ts2'}}, + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 " \ + "(c1 integer NOT NULL, c2 text) TABLESPACE ts1" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_pkey PRIMARY KEY (c1) USING INDEX TABLESPACE ts2" + + def test_create_index(self): + "Create an index using a tablespace" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts2', + 'unique': True}}, + 'tablespace': 'ts1'}}) + sql = self.to_sql(inmap, [CREATE_TABLE]) + assert fix_indent(sql[0]) == "CREATE UNIQUE INDEX t1_idx " \ + "ON sd.t1 (c1) TABLESPACE ts2" + + def test_move_index(self): + "Move a index from one tablespace to another" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}], + 'indexes': {'t1_idx': {'keys': ['c1'], 'tablespace': 'ts2'}}}}) + stmts = ["CREATE TABLE t1 (c1 integer, c2 text)", + "CREATE INDEX t1_idx ON t1 (c1) TABLESPACE ts1"] + sql = self.to_sql(inmap, stmts) + assert fix_indent(sql[0]) == "ALTER INDEX sd.t1_idx SET TABLESPACE ts2" diff --git a/tests/dbobject/test_textsearch.py b/tests/dbobject/test_textsearch.py new file mode 100644 index 0000000..3093cef --- /dev/null +++ b/tests/dbobject/test_textsearch.py @@ -0,0 +1,276 @@ +# -*- coding: utf-8 -*- +"""Test text search objects""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_TSC_STMT = "CREATE TEXT SEARCH CONFIGURATION sd.tsc1 (PARSER = tsp1)" +CREATE_TSD_STMT = "CREATE TEXT SEARCH DICTIONARY sd.tsd1 (TEMPLATE = simple, "\ + "stopwords = 'english')" +CREATE_TSP_STMT = "CREATE TEXT SEARCH PARSER sd.tsp1 (START = prsd_start, " \ + "GETTOKEN = prsd_nexttoken, END = prsd_end, LEXTYPES = prsd_lextype, " \ + "HEADLINE = prsd_headline)" +CREATE_TST_STMT = "CREATE TEXT SEARCH TEMPLATE sd.tst1 (INIT = dsimple_init, "\ + "LEXIZE = dsimple_lexize)" +COMMENT_TSC_STMT = "COMMENT ON TEXT SEARCH CONFIGURATION sd.tsc1 IS " \ + "'Test configuration tsc1'" +COMMENT_TSD_STMT = "COMMENT ON TEXT SEARCH DICTIONARY sd.tsd1 IS " \ + "'Test dictionary tsd1'" +COMMENT_TSP_STMT = "COMMENT ON TEXT SEARCH PARSER sd.tsp1 IS " \ + "'Test parser tsp1'" +COMMENT_TST_STMT = "COMMENT ON TEXT SEARCH TEMPLATE sd.tst1 IS " \ + "'Test template tst1'" + + +class TextSearchConfigToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search configurations""" + + superuser = True + + def test_map_ts_config(self): + "Map an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'] == { + 'parser': 'tsp1'} + + def test_map_cross_schema_ts_config(self): + "Map a text search config with parser in different schema" + stmts = ["CREATE SCHEMA s1", + "CREATE TEXT SEARCH PARSER s1.tsp1 " + "(START = prsd_start, GETTOKEN = prsd_nexttoken, " + "END = prsd_end, LEXTYPES = prsd_lextype)", + "CREATE TEXT SEARCH CONFIGURATION tsc1 (PARSER = s1.tsp1)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'] == { + 'parser': 's1.tsp1'} + + def test_map_ts_config_comment(self): + "Map a text search configuration with a comment" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT, COMMENT_TSC_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search configuration tsc1'][ + 'description'] == 'Test configuration tsc1' + + +class TextSearchConfigToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search configurations""" + + def test_create_ts_config(self): + "Create a text search configuration that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}, 'text search configuration tsc1': { + 'parser': 'tsp1'}}) + sql = self.to_sql(inmap, [CREATE_TSP_STMT]) + assert fix_indent(sql[0]) == CREATE_TSC_STMT + + def test_create_ts_config_in_schema(self): + "Create a text search config with parser in non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype'}}}) + inmap['schema sd'].update({'text search configuration tsc1': { + 'parser': 's1.tsp1'}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == "CREATE TEXT SEARCH PARSER s1.tsp1 " \ + "(START = prsd_start, GETTOKEN = prsd_nexttoken, " \ + "END = prsd_end, LEXTYPES = prsd_lextype)" + assert fix_indent(sql[1]) == \ + "CREATE TEXT SEARCH CONFIGURATION sd.tsc1 (PARSER = s1.tsp1)" + + def test_bad_map_ts_config_(self): + "Error creating a text search configuration with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsc1': {'parser': 'tsp1'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_config(self): + "Drop an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql[0] == "DROP TEXT SEARCH CONFIGURATION sd.tsc1" + assert sql[1] == "DROP TEXT SEARCH PARSER sd.tsp1" + + def test_comment_on_ts_config(self): + "Create a comment for an existing text search configuration" + stmts = [CREATE_TSP_STMT, CREATE_TSC_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'text search configuration tsc1': { + 'parser': 'tsp1', 'description': "Test configuration tsc1"}, + 'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}}) + sql = self.to_sql(inmap, stmts, superuser=True) + assert sql == [COMMENT_TSC_STMT] + + +class TextSearchDictToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search dictionaries""" + + def test_map_ts_dict(self): + "Map an existing text search dictionary" + dbmap = self.to_map([CREATE_TSD_STMT]) + assert dbmap['schema sd']['text search dictionary tsd1'] == { + 'template': 'simple', 'options': "stopwords = 'english'"} + + def test_map_ts_dict_comment(self): + "Map a text search dictionary with a comment" + stmts = [CREATE_TSD_STMT, COMMENT_TSD_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search dictionary tsd1'][ + 'description'], 'Test dictionary tsd1' + + +class TextSearchDictToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search dictionaries""" + + def test_create_ts_dict(self): + "Create a text search dictionary that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search dictionary tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TSD_STMT + + def test_bad_map_ts_dict(self): + "Error creating a text search dictionary with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'"}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_dict(self): + "Drop an existing text search dictionary" + sql = self.to_sql(self.std_map(), [CREATE_TSD_STMT]) + assert sql == ["DROP TEXT SEARCH DICTIONARY sd.tsd1"] + + def test_comment_on_ts_dict(self): + "Create a comment for an existing text search dictionary" + inmap = self.std_map() + inmap['schema sd'].update({'text search dictionary tsd1': { + 'template': 'simple', 'options': "stopwords = 'english'", + 'description': "Test dictionary tsd1"}}) + sql = self.to_sql(inmap, [CREATE_TSD_STMT]) + assert sql == [COMMENT_TSD_STMT] + + +class TextSearchParserToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search parsers""" + + superuser = True + + def test_map_ts_parser(self): + "Map an existing text search parser" + stmts = [CREATE_TSP_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search parser tsp1'] == { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'} + + def test_map_ts_parser_comment(self): + "Map a text search parser with a comment" + stmts = [CREATE_TSP_STMT, COMMENT_TSP_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search parser tsp1'][ + 'description'] == 'Test parser tsp1' + + +class TextSearchParserToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search parsers""" + + def test_create_ts_parser(self): + "Create a text search parser that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TSP_STMT + + def test_bad_map_ts_parser(self): + "Error creating a text search parser with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_parser(self): + "Drop an existing text search parser" + sql = self.to_sql(self.std_map(), [CREATE_TSP_STMT], superuser=True) + assert sql == ["DROP TEXT SEARCH PARSER sd.tsp1"] + + def test_comment_on_ts_parser(self): + "Create a comment for an existing text search parser" + inmap = self.std_map() + inmap['schema sd'].update({'text search parser tsp1': { + 'start': 'prsd_start', 'gettoken': 'prsd_nexttoken', + 'end': 'prsd_end', 'lextypes': 'prsd_lextype', + 'headline': 'prsd_headline', 'description': "Test parser tsp1"}}) + sql = self.to_sql(inmap, [CREATE_TSP_STMT], superuser=True) + assert sql == [COMMENT_TSP_STMT] + + +class TextSearchTemplateToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing text search templates""" + + superuser = True + + def test_map_ts_template(self): + "Map an existing text search template" + dbmap = self.to_map([CREATE_TST_STMT]) + assert dbmap['schema sd']['text search template tst1'] == { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'} + + def test_map_ts_template_comment(self): + "Map a text search template with a comment" + stmts = [CREATE_TST_STMT, COMMENT_TST_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['text search template tst1'][ + 'description'], 'Test template tst1' + + +class TextSearchTemplateToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation for input text search templates""" + + def test_create_ts_template(self): + "Create a text search template that didn't exist" + inmap = self.std_map() + inmap['schema sd'].update({'text search template tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'}}) + sql = self.to_sql(inmap, superuser=True) + assert fix_indent(sql[0]) == CREATE_TST_STMT + + def test_bad_map_ts_template(self): + "Error creating a text search template with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize'}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_ts_template(self): + "Drop an existing text search template" + sql = self.to_sql(self.std_map(), [CREATE_TST_STMT], superuser=True) + assert sql == ["DROP TEXT SEARCH TEMPLATE sd.tst1"] + + def test_comment_on_ts_template(self): + "Create a comment for an existing text search template" + inmap = self.std_map() + inmap['schema sd'].update({'text search template tst1': { + 'init': 'dsimple_init', 'lexize': 'dsimple_lexize', + 'description': "Test template tst1"}}) + sql = self.to_sql(inmap, [CREATE_TST_STMT], superuser=True) + assert sql == [COMMENT_TST_STMT] diff --git a/tests/dbobject/test_trigger.py b/tests/dbobject/test_trigger.py new file mode 100644 index 0000000..056a7c4 --- /dev/null +++ b/tests/dbobject/test_trigger.py @@ -0,0 +1,488 @@ +# -*- coding: utf-8 -*- +"""Test triggers""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +FUNC_SRC = "BEGIN NEW.c3 := CURRENT_DATE; RETURN NEW; END" +FUNC_INSTEAD_SRC = "BEGIN INSERT INTO t1 VALUES (NEW.c1, NEW.c2, now()); " \ + "RETURN NULL; END" +FUNC_REF_NEW_SRC = "BEGIN SELECT SUM(c2) FROM ins; RETURN NULL; END" +FUNC_REF_NEW_OLD_SRC = "BEGIN SELECT SUM(c2) FROM del; " \ + "SELECT SUM(c2) FROM ins; RETURN NULL; END" +CREATE_TABLE_STMT = "CREATE TABLE sd.t1 (c1 integer, c2 text, " \ + "c3 date)" +CREATE_TABLE_STMT2 = "CREATE TABLE t1 (c1 integer, c2 text, " \ + "c3 text, tsidx tsvector)" +CREATE_TABLE_STMT3 = "CREATE TABLE sd.t1 (c1 integer, c2 money)" +CREATE_FUNC_STMT = "CREATE FUNCTION sd.f1() RETURNS trigger LANGUAGE plpgsql" \ + " AS $_$%s$_$" % FUNC_SRC +CREATE_STMT = "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON sd.t1 " \ + "FOR EACH ROW EXECUTE PROCEDURE sd.f1()" +COMMENT_STMT = "COMMENT ON TRIGGER tr1 ON sd.t1 IS 'Test trigger tr1'" + + +class TriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing triggers""" + + def test_map_trigger1(self): + "Map a simple trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}} + + def test_map_trigger2(self): + "Map another simple trigger with different attributes" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER DELETE OR TRUNCATE ON t1 " + "EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['delete', 'truncate'], + 'level': 'statement', 'procedure': 'sd.f1'}} + + def test_map_trigger_update_cols(self): + "Map trigger with UPDATE OF columns" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER INSERT OR UPDATE OF c1, c2 ON t1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['insert', 'update'], + 'columns': ['c1', 'c2'], 'level': 'row', + 'procedure': 'sd.f1'}} + + def test_map_trigger_conditional(self): + "Map trigger with a WHEN qualification" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE TRIGGER tr1 AFTER UPDATE ON t1 FOR EACH ROW " + "WHEN (OLD.c2 IS DISTINCT FROM NEW.c2) " + "EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'procedure': 'sd.f1', + 'condition': '(old.c2 IS DISTINCT FROM new.c2)'}} + + def test_map_trigger_instead(self): + "Map an INSTEAD OF trigger" + stmts = [CREATE_TABLE_STMT, "CREATE VIEW v1 AS SELECT c1, c2 FROM t1", + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_INSTEAD_SRC, + "CREATE TRIGGER tr1 INSTEAD OF INSERT ON v1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['view v1']['triggers'] == { + 'tr1': {'timing': 'instead of', 'events': ['insert'], + 'level': 'row', 'procedure': 'sd.f1'}} + + def test_map_tsvector_trigger(self): + "Map a text search (tsvector) trigger" + stmts = [ + CREATE_TABLE_STMT2, + "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON sd.t1 " + "FOR EACH ROW EXECUTE PROCEDURE " + "tsvector_update_trigger('tsidx', 'pg_catalog.english', 'c2')"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', + 'procedure': {'name': 'tsvector_update_trigger', + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2'"}}} + + def test_map_trigger_function_distinct_schemas(self): + "Map a trigger in a non-default schema with function in different one" + stmts = ["CREATE SCHEMA s1", "CREATE TABLE s1.t1 (c1 integer, " + "c2 text, c3 date)", "CREATE SCHEMA s2", + "CREATE FUNCTION s2.f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_SRC, + "CREATE TRIGGER tr1 BEFORE INSERT OR UPDATE ON s1.t1 " + "FOR EACH ROW EXECUTE PROCEDURE s2.f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema s1']['table t1']['triggers'] == { + 'tr1': {'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 's2.f1'}} + + def test_map_trigger_referencing_new(self): + "Map a trigger that uses REFERENCING NEW TABLE" + stmts = [CREATE_TABLE_STMT3, + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_REF_NEW_SRC, + "CREATE TRIGGER tr1 AFTER INSERT ON t1 " + "REFERENCING NEW TABLE AS ins " + "FOR EACH STATEMENT EXECUTE FUNCTION f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['insert'], + 'level': 'statement', 'procedure': 'sd.f1', + 'referencing_new': 'ins'}} + + def test_map_trigger_referencing_new_old(self): + "Map a trigger that uses REFERENCING NEW and OLD TABLE" + stmts = [CREATE_TABLE_STMT3, + "CREATE FUNCTION f1() RETURNS trigger LANGUAGE plpgsql AS " + "$_$%s$_$" % FUNC_REF_NEW_OLD_SRC, + "CREATE TRIGGER tr1 AFTER UPDATE ON t1 " + "REFERENCING NEW TABLE AS ins OLD TABLE AS del " + "FOR EACH ROW EXECUTE FUNCTION f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'procedure': 'sd.f1', + 'referencing_new': 'ins', 'referencing_old': 'del'}} + + def test_map_trigger_comment(self): + "Map a trigger comment" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers']['tr1'][ + 'description'] == 'Test trigger tr1' + + +class ConstraintTriggerToMapTestCase(DatabaseToMapTestCase): + """Test mapping of existing constraint triggers""" + + def test_map_trigger(self): + "Map a simple constraint trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE CONSTRAINT TRIGGER tr1 AFTER INSERT OR UPDATE ON t1 " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'constraint': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}} + + def test_map_trigger_deferrable(self): + "Map a deferrable, initially deferred constraint trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, + "CREATE CONSTRAINT TRIGGER tr1 AFTER INSERT OR UPDATE ON t1 " + "DEFERRABLE INITIALLY DEFERRED " + "FOR EACH ROW EXECUTE PROCEDURE f1()"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['table t1']['triggers'] == { + 'tr1': {'constraint': True, 'deferrable': True, + 'initially_deferred': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}} + + +class TriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_trigger1(self): + "Create a simple trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == CREATE_STMT + + def test_create_trigger2(self): + "Create another simple trigger with" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'after', + 'events': ['delete', 'truncate'], + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 AFTER DELETE OR " \ + "TRUNCATE ON sd.t1 FOR EACH STATEMENT EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_update_cols(self): + "Create a trigger with UPDATE OF columns" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'before', 'events': [ + 'insert', 'update'], 'columns': ['c1', 'c2'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE INSERT OR " \ + "UPDATE OF c1, c2 ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_conditional(self): + "Create a trigger with a WHEN qualification" + inmap = self.std_map() + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': {'timing': 'before', 'events': [ + 'update'], 'level': 'row', 'procedure': 'sd.f1', + 'condition': '(old.c2 IS DISTINCT FROM new.c2)'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE UPDATE " \ + "ON sd.t1 FOR EACH ROW WHEN ((old.c2 IS DISTINCT FROM new.c2)) " \ + "EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_instead(self): + "Create an INSTEAD OF trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', + 'source': FUNC_INSTEAD_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}, + 'view v1': {'definition': "SELECT c1, c2 FROM t1", + 'triggers': {'tr1': {'timing': 'instead of', + 'events': ['insert'], + 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT + cr1, cr2 = (1, 2) if 'VIEW' in sql[1] else (2, 1) + assert fix_indent(sql[cr1]) == \ + "CREATE VIEW sd.v1 AS SELECT c1, c2 FROM t1" + assert fix_indent(sql[cr2]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_INSTEAD_SRC + assert fix_indent(sql[3]) == "CREATE TRIGGER tr1 INSTEAD OF INSERT " \ + "ON sd.v1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_add_tsvector_trigger(self): + "Add a text search (tsvector) trigger" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}, + {'tsidx': {'type': 'tsvector'}}], + 'triggers': {'t1_tsidx_update': { + 'timing': 'before', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': {'name': 'tsvector_update_trigger', + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2'"}}}}}) + sql = self.to_sql(inmap, [CREATE_TABLE_STMT2]) + assert fix_indent(sql[0]) == "CREATE TRIGGER t1_tsidx_update BEFORE" \ + " INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE " \ + "tsvector_update_trigger('tsidx', 'pg_catalog.english', 'c2')" + + def test_change_tsvector_trigger(self): + "Change a text search (tsvector) trigger" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'text'}}, + {'tsidx': {'type': 'tsvector'}}], + 'triggers': {'t1_tsidx_update': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', + 'procedure': {'name': "tsvector_update_trigger", + 'arguments': + "'tsidx', 'pg_catalog.english', 'c2', 'c3'"}}}}}) + stmts = [CREATE_TABLE_STMT2, + "CREATE TRIGGER t1_tsidx_update BEFORE INSERT OR UPDATE ON " + "t1 FOR EACH ROW EXECUTE PROCEDURE tsvector_update_trigger" + "('tsidx', 'pg_catalog.english', 'c2')"] + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP TRIGGER t1_tsidx_update ON sd.t1" + assert fix_indent(sql[1]) == "CREATE TRIGGER t1_tsidx_update BEFORE" \ + " INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE " \ + "tsvector_update_trigger('tsidx', 'pg_catalog.english', " \ + "'c2', 'c3')" + + def test_create_trigger_function_distinct_schemas(self): + "Create a trigger in non-default schema with function in different one" + inmap = self.std_map(plpgsql_installed=True) + inmap.update({'schema s2': {'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}, + 'schema s1': { + 'table t1': { + 'columns': [ + {'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', + 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 's2.f1'}}}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1", "CREATE SCHEMA s2"]) + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 BEFORE INSERT OR " \ + "UPDATE ON s1.t1 FOR EACH ROW EXECUTE PROCEDURE s2.f1()" + + def test_create_trigger_referencing(self): + "Create a trigger that uses REFERENCING OLD and NEW" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', + 'source': FUNC_REF_NEW_OLD_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'money'}}], + 'triggers': {'tr1': {'timing': 'after', 'events': ['update'], + 'level': 'row', 'referencing_new': 'ins', + 'referencing_old': 'del', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TABLE_STMT3 + assert fix_indent(sql[1]) == "CREATE FUNCTION sd.f1() RETURNS " \ + "trigger LANGUAGE plpgsql AS $_$%s$_$" % FUNC_REF_NEW_OLD_SRC + assert fix_indent(sql[2]) == "CREATE TRIGGER tr1 AFTER UPDATE " \ + "ON sd.t1 REFERENCING NEW TABLE AS ins OLD TABLE AS del " \ + "FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_drop_trigger(self): + "Drop an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}]}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["DROP TRIGGER tr1 ON sd.t1"] + + def test_drop_trigger_table(self): + "Drop an existing trigger and the related table" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + sql = self.to_sql(inmap, stmts) + assert sql[0] == "DROP TRIGGER tr1 ON sd.t1" + assert sql[1] == "DROP TABLE sd.t1" + + def test_trigger_with_comment(self): + "Create a trigger with a comment" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Test trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == CREATE_STMT + assert sql[3] == COMMENT_STMT + + def test_comment_on_trigger(self): + "Create a comment on an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Test trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [COMMENT_STMT] + + def test_drop_trigger_comment(self): + "Drop a comment on an existing trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON TRIGGER tr1 ON sd.t1 IS NULL"] + + def test_change_trigger_comment(self): + "Change existing comment on a trigger" + stmts = [CREATE_TABLE_STMT, CREATE_FUNC_STMT, CREATE_STMT, + COMMENT_STMT] + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'description': 'Changed trigger tr1', + 'timing': 'before', 'events': ['insert', 'update'], + 'level': 'row', 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap, stmts) + assert sql == [ + "COMMENT ON TRIGGER tr1 ON sd.t1 IS 'Changed trigger tr1'"] + + +class ConstraintTriggerToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input triggers""" + + def test_create_trigger(self): + "Create a constraint trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'constraint': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE CONSTRAINT TRIGGER tr1 AFTER " \ + "INSERT OR UPDATE ON sd.t1 FOR EACH ROW EXECUTE PROCEDURE sd.f1()" + + def test_create_trigger_deferrable(self): + "Create a deferrable constraint trigger" + inmap = self.std_map(plpgsql_installed=True) + inmap['schema sd'].update({'function f1()': { + 'language': 'plpgsql', 'returns': 'trigger', 'source': FUNC_SRC}}) + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'date'}}], + 'triggers': {'tr1': { + 'constraint': True, 'deferrable': True, + 'initially_deferred': True, 'timing': 'after', + 'events': ['insert', 'update'], 'level': 'row', + 'procedure': 'sd.f1'}}}}) + sql = self.to_sql(inmap) + crt0, crt1 = (0, 1) if 'TABLE' in sql[0] else (1, 0) + assert fix_indent(sql[crt0]) == CREATE_TABLE_STMT + assert fix_indent(sql[crt1]) == CREATE_FUNC_STMT + assert fix_indent(sql[2]) == "CREATE CONSTRAINT TRIGGER tr1 " \ + "AFTER INSERT OR UPDATE ON sd.t1 DEFERRABLE INITIALLY " \ + "DEFERRED FOR EACH ROW EXECUTE PROCEDURE sd.f1()" diff --git a/tests/dbobject/test_type.py b/tests/dbobject/test_type.py new file mode 100644 index 0000000..b7a20e1 --- /dev/null +++ b/tests/dbobject/test_type.py @@ -0,0 +1,250 @@ +# -*- coding: utf-8 -*- +"""Test enums and other types""" + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_COMPOSITE_STMT = "CREATE TYPE sd.t1 AS " \ + "(x integer, y integer, z integer)" +CREATE_ENUM_STMT = "CREATE TYPE sd.t1 AS ENUM ('red', 'green', 'blue')" +CREATE_SHELL_STMT = "CREATE TYPE sd.t1" +CREATE_RANGE_STMT = "CREATE TYPE sd.t1 AS RANGE (SUBTYPE = smallint)" +CREATE_FUNC_IN = "CREATE FUNCTION sd.t1textin(cstring) RETURNS t1 " \ + "LANGUAGE internal IMMUTABLE STRICT AS $$textin$$" +CREATE_FUNC_OUT = "CREATE FUNCTION sd.t1textout(sd.t1) RETURNS cstring " \ + "LANGUAGE internal IMMUTABLE STRICT AS $$textout$$" +CREATE_TYPE_STMT = "CREATE TYPE t1 (INPUT = t1textin, OUTPUT = t1textout)" +COMMENT_STMT = "COMMENT ON TYPE t1 IS 'Test type t1'" + + +class CompositeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created composite types""" + + def test_composite(self): + "Map a composite type" + dbmap = self.to_map([CREATE_COMPOSITE_STMT]) + assert dbmap['schema sd']['type t1'] == { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]} + + def test_dropped_attribute(self): + "Map a composite type which has a dropped attribute" + stmts = [CREATE_COMPOSITE_STMT, "ALTER TYPE t1 DROP ATTRIBUTE y"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]} + + +class CompositeToSqlTestCase(InputMapToSqlTestCase): + """Test creation and modification of composite types""" + + def test_create_composite(self): + "Create a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_COMPOSITE_STMT + + def test_drop_composite(self): + "Drop an existing composite" + sql = self.to_sql(self.std_map(), [CREATE_COMPOSITE_STMT]) + assert sql == ["DROP TYPE sd.t1"] + + def test_rename_composite(self): + "Rename an existing composite" + inmap = self.std_map() + inmap['schema sd'].update({'type t2': { + 'oldname': 't1', + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert sql == ["ALTER TYPE sd.t1 RENAME TO t2"] + + def test_add_attribute(self): + "Add an attribute to a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, ["CREATE TYPE t1 AS (x integer, y integer)"]) + assert fix_indent(sql[0]) == "ALTER TYPE sd.t1 ADD ATTRIBUTE z integer" + + def test_drop_attribute(self): + "Drop an attribute from a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert fix_indent(sql[0]) == "ALTER TYPE sd.t1 DROP ATTRIBUTE y" + + def test_drop_attribute_schema(self): + "Drop an attribute from a composite type within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'z': {'type': 'integer'}}]}}}) + sql = self.to_sql(inmap, [ + "CREATE SCHEMA s1", + "CREATE TYPE s1.t1 AS (x integer, y integer, z integer)"]) + assert fix_indent(sql[0]) == "ALTER TYPE s1.t1 DROP ATTRIBUTE y" + + def test_rename_attribute(self): + "Rename an attribute of a composite type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'attributes': [{'x': {'type': 'integer'}}, + {'y1': {'type': 'integer', 'oldname': 'y'}}, + {'z': {'type': 'integer'}}]}}) + sql = self.to_sql(inmap, [CREATE_COMPOSITE_STMT]) + assert fix_indent(sql[0]) == \ + "ALTER TYPE sd.t1 RENAME ATTRIBUTE y TO y1" + + +class EnumToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created enum types""" + + def test_enum(self): + "Map an enum" + dbmap = self.to_map([CREATE_ENUM_STMT]) + assert dbmap['schema sd']['type t1'] == { + 'labels': ['red', 'green', 'blue']} + + +class EnumToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input enums""" + + def test_create_enum(self): + "Create an enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'labels': ['red', 'green', 'blue']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_ENUM_STMT + + def test_drop_enum(self): + "Drop an existing enum" + sql = self.to_sql(self.std_map(), [CREATE_ENUM_STMT]) + assert sql == ["DROP TYPE sd.t1"] + + def test_change_enum(self): + "Change an existing enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'labels': ['red', 'yellow', 'blue']}}) + sql = self.to_sql(inmap, [CREATE_ENUM_STMT]) + assert sql[0] == "DROP TYPE sd.t1" + assert fix_indent(sql[1]) == \ + "CREATE TYPE sd.t1 AS ENUM ('red', 'yellow', 'blue')" + + def test_rename_enum(self): + "Rename an existing enum" + inmap = self.std_map() + inmap['schema sd'].update({'type t2': { + 'oldname': 't1', 'labels': ['red', 'green', 'blue']}}) + sql = self.to_sql(inmap, [CREATE_ENUM_STMT]) + assert sql == ["ALTER TYPE sd.t1 RENAME TO t2"] + + +class BaseTypeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created base type types""" + + superuser = True + + def test_base_type(self): + "Map a base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + CREATE_TYPE_STMT] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain', 'category': 'U'} + + def test_base_type_category(self): + "Map a base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + "CREATE TYPE t1 (INPUT = t1textin, OUTPUT = t1textout, " + "CATEGORY = 'S')"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain', 'category': 'S'} + + +class BaseTypeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input base types""" + + def test_create_base_type(self): + "Create a base type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'input': 't1textin', 'output': 't1textout', + 'internallength': 'variable', 'alignment': 'int4', + 'storage': 'plain'}, 'function t1textin(cstring)': { + 'language': 'internal', 'returns': 't1', 'strict': True, + 'volatility': 'immutable', 'source': 'textin'}, + 'function t1textout(sd.t1)': { + 'language': 'internal', 'returns': 'cstring', + 'strict': True, 'volatility': 'immutable', + 'source': 'textout'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_SHELL_STMT + assert fix_indent(sql[1]) == CREATE_FUNC_IN + assert fix_indent(sql[2]) == CREATE_FUNC_OUT + assert fix_indent(sql[3]) == "CREATE TYPE sd.t1 (INPUT = t1textin, " \ + "OUTPUT = t1textout, INTERNALLENGTH = variable, " \ + "ALIGNMENT = int4, STORAGE = plain)" + + def test_drop_type(self): + "Drop an existing base type" + stmts = [CREATE_SHELL_STMT, CREATE_FUNC_IN, CREATE_FUNC_OUT, + CREATE_TYPE_STMT] + sql = self.to_sql(self.std_map(), stmts, superuser=True) + assert sql == ["DROP TYPE sd.t1 CASCADE"] + + +class RangeToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created RANGE types""" + + def test_range_simple(self): + "Map a simple range type" + dbmap = self.to_map([CREATE_RANGE_STMT]) + assert dbmap['schema sd']['type t1'] == {'subtype': 'int2'} + + def test_range_subtypediff(self): + "Map a range type with a subtype difference function" + stmts = ["CREATE TYPE t1 AS RANGE (SUBTYPE = float8, " + "SUBTYPE_DIFF = float8mi)"] + dbmap = self.to_map(stmts) + assert dbmap['schema sd']['type t1'] == { + 'subtype': 'float8', 'subtype_diff': 'float8mi'} + + +class RangeToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input range types""" + + def test_create_range_simple(self): + "Create a range type" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': {'subtype': 'smallint'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_RANGE_STMT + + def test_create_range_subtype_diff(self): + "Create a range with a subtype diff function" + inmap = self.std_map() + inmap['schema sd'].update({'type t1': { + 'subtype': 'float8', 'subtype_diff': 'float8mi'}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == ( + "CREATE TYPE sd.t1 AS RANGE (SUBTYPE = float8, " + "SUBTYPE_DIFF = float8mi)") diff --git a/tests/dbobject/test_view.py b/tests/dbobject/test_view.py new file mode 100644 index 0000000..4ea97a0 --- /dev/null +++ b/tests/dbobject/test_view.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +"""Test views""" + +import pytest + +from pyrseas.testutils import DatabaseToMapTestCase +from pyrseas.testutils import InputMapToSqlTestCase, fix_indent + +CREATE_STMT = "CREATE VIEW sd.v1 AS SELECT now()::date AS today" +CREATE_TBL = "CREATE TABLE sd.t1 (c1 integer, c2 text, c3 integer)" +CREATE_STMT2 = "CREATE VIEW sd.v1 AS SELECT c1, c3 * 2 AS c2 FROM t1" +COMMENT_STMT = "COMMENT ON VIEW sd.v1 IS 'Test view v1'" +VIEW_DEFN = " SELECT now()::date AS today;" + + +class ViewToMapTestCase(DatabaseToMapTestCase): + """Test mapping of created views""" + + def test_map_view_no_table(self): + "Map a created view without a table dependency" + dbmap = self.to_map([CREATE_STMT]) + expmap = {'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_table(self): + "Map a created view with a table dependency" + stmts = [CREATE_TBL, CREATE_STMT2] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'integer'}}], + 'depends_on': ['table t1'], + 'definition': " SELECT t1.c1," + "\n t1.c3 * 2 AS c2\n FROM sd.t1;"} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_columns(self): + "Map a complex view's columns in addition to its definition" + stmts = ["CREATE TABLE t1 (c1 INTEGER UNIQUE)", + "CREATE TABLE t2 (c2 INTEGER PRIMARY KEY REFERENCES t1(c1))", + "CREATE VIEW v1 AS SELECT (ROW(t1.*)::t2).*, t1, 5 AS const " + "FROM t1"] + dbmap = self.to_map(stmts) + expmap = {'columns': [{'c2': {'type': 'integer'}}, + {'t1': {'type': 'sd.t1'}}, + {'const': {'type': 'integer'}}], + 'definition': " SELECT (ROW(t1.c1)::sd.t2).c2 AS c2," + "\n t1.*::sd.t1 AS t1,\n 5 AS const\n FROM sd.t1;", + 'depends_on': ['table t1']} + assert dbmap['schema sd']['view v1'] == expmap + + def test_map_view_comment(self): + "Map a view with a comment" + dbmap = self.to_map([CREATE_STMT, COMMENT_STMT]) + assert dbmap['schema sd']['view v1']['description'] == \ + 'Test view v1' + + +class ViewToSqlTestCase(InputMapToSqlTestCase): + """Test SQL generation from input views""" + + def test_create_view_no_table(self): + "Create a view with no table dependency" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': {'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + + def test_create_view(self): + "Create a view" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, {'c2': {'type': 'text'}}, + {'c3': {'type': 'integer'}}]}}) + inmap['schema sd'].update({'view v1': { + 'definition': "SELECT c1, c3 * 2 AS c2 FROM t1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_TBL + assert fix_indent(sql[1]) == CREATE_STMT2 + + def test_create_view_in_schema(self): + "Create a view within a non-default schema" + inmap = self.std_map() + inmap.update({'schema s1': {'view v1': {'definition': VIEW_DEFN}}}) + sql = self.to_sql(inmap, ["CREATE SCHEMA s1"]) + assert fix_indent(sql[0]) == \ + "CREATE VIEW s1.v1 AS SELECT now()::date AS today" + + def test_bad_view_map(self): + "Error creating a view with a bad map" + inmap = self.std_map() + inmap['schema sd'].update({'v1': {'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap) + + def test_drop_view_no_table(self): + "Drop an existing view without a table dependency" + sql = self.to_sql(self.std_map(), [CREATE_STMT]) + assert sql == ["DROP VIEW sd.v1"] + + def test_drop_view(self): + "Drop an existing view with table dependencies" + stmts = ["CREATE TABLE t1 (c1 INTEGER, c2 TEXT)", + "CREATE TABLE t2 (c1 INTEGER, c3 TEXT)", + "CREATE VIEW v1 AS SELECT t1.c1, c2, c3 " + "FROM t1 JOIN t2 ON (t1.c1 = t2.c1)"] + sql = self.to_sql(self.std_map(), stmts) + assert sql[0] == "DROP VIEW sd.v1" + # can't control which table will be dropped first + drt1 = 1 + drt2 = 2 + if 't1' in sql[2]: + drt1 = 2 + drt2 = 1 + assert sql[drt1] == "DROP TABLE sd.t1" + assert sql[drt2] == "DROP TABLE sd.t2" + + def test_rename_view(self): + "Rename an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v2': { + 'oldname': 'v1', 'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == ["ALTER VIEW sd.v1 RENAME TO v2"] + + def test_bad_rename_view(self): + "Error renaming a non-existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v2': { + 'oldname': 'v3', 'definition': VIEW_DEFN}}) + with pytest.raises(KeyError): + self.to_sql(inmap, [CREATE_STMT]) + + def test_change_view_defn(self): + "Change view definition" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': " SELECT 'now'::text::date AS today;"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert fix_indent(sql[0]) == "CREATE OR REPLACE VIEW sd.v1 AS " \ + "SELECT 'now'::text::date AS today" + + def test_change_column_name(self): + "Attempt rename view column name (disallowed)" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'todays_date': {'type': 'date'}}], + 'definition': " SELECT now()::date AS todays_date;"}}) + with pytest.raises(KeyError): + sql = self.to_sql(inmap, [CREATE_STMT]) + + def test_change_column_type(self): + "Change view column type to different type (disallowed)" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'numeric'}}], + 'definition': " SELECT c1, c3 * 2.0 AS c2 FROM t1;"}}) + with pytest.raises(TypeError): + sql = self.to_sql(inmap, [CREATE_TBL, CREATE_STMT2]) + + def test_view_depend_pk(self): + "Create a view that depends on a primary key. See issue #72" + inmap = self.std_map() + inmap['schema sd'].update({'table t1': { + 'columns': [{'c1': {'type': 'integer'}}, + {'c2': {'type': 'text'}}], + 'primary_key': {'t1_pkey': {'columns': ['c1']}}}, + 'view v1': { + 'definition': " SELECT t1.c1,\n t1.c2\n FROM t1\n " + "GROUP BY t1.c1;", + 'depends_on': ['table t1']}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == "CREATE TABLE sd.t1 (c1 integer, c2 text)" + assert fix_indent(sql[1]) == "ALTER TABLE sd.t1 ADD CONSTRAINT " \ + "t1_pkey PRIMARY KEY (c1)" + assert fix_indent(sql[2]) == "CREATE VIEW sd.v1 AS SELECT t1.c1, " \ + "t1.c2 FROM t1 GROUP BY t1.c1" + + def test_view_with_comment(self): + "Create a view with a comment" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'definition': VIEW_DEFN, 'description': "Test view v1"}}) + sql = self.to_sql(inmap) + assert fix_indent(sql[0]) == CREATE_STMT + assert sql[1] == COMMENT_STMT + + def test_comment_on_view(self): + "Create a comment for an existing view" + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN, 'description': "Test view v1"}}) + sql = self.to_sql(inmap, [CREATE_STMT]) + assert sql == [COMMENT_STMT] + + def test_drop_view_comment(self): + "Drop the comment on an existing view" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON VIEW sd.v1 IS NULL"] + + def test_change_view_comment(self): + "Change existing comment on a view" + stmts = [CREATE_STMT, COMMENT_STMT] + inmap = self.std_map() + inmap['schema sd'].update({'view v1': { + 'columns': [{'today': {'type': 'date'}}], + 'definition': VIEW_DEFN, 'description': "Changed view v1"}}) + sql = self.to_sql(inmap, stmts) + assert sql == ["COMMENT ON VIEW sd.v1 IS 'Changed view v1'"] diff --git a/tests/functional/__init__.py b/tests/functional/__init__.py new file mode 100644 index 0000000..5c20503 --- /dev/null +++ b/tests/functional/__init__.py @@ -0,0 +1 @@ +"""Pyrseas functional tests""" diff --git a/tests/functional/autodoc-schema.sql b/tests/functional/autodoc-schema.sql new file mode 100644 index 0000000..80f31b9 --- /dev/null +++ b/tests/functional/autodoc-schema.sql @@ -0,0 +1,125 @@ +-- +-- $Id: regressdatabase.sql,v 1.2 2006/02/13 01:15:56 rbt Exp $ +-- + +BEGIN; +-- +-- Foreign key'd structure, check constraints, primary keys +-- and duplicate table names in different schemas +-- +CREATE SCHEMA product + CREATE TABLE product + ( product_id SERIAL PRIMARY KEY + , product_code text NOT NULL UNIQUE + CHECK(product_code = upper(product_code)) + , product_description text + ); + +CREATE SCHEMA store + CREATE TABLE store + ( store_id SERIAL PRIMARY KEY + , store_code text NOT NULL UNIQUE + CHECK(store_code = upper(store_code)) + , store_description text + ) + + CREATE TABLE inventory + ( store_id integer REFERENCES store + ON UPDATE CASCADE ON DELETE RESTRICT + , product_id integer REFERENCES product.product + ON UPDATE CASCADE ON DELETE RESTRICT + , PRIMARY KEY(store_id, product_id) + , quantity integer NOT NULL CHECK(quantity > 0) + ); + +-- +-- Another schema with +-- +CREATE SCHEMA warehouse + CREATE TABLE warehouse + ( warehouse_id SERIAL PRIMARY KEY + , warehouse_code text NOT NULL UNIQUE + CHECK(warehouse_code = upper(warehouse_code)) + , warehouse_manager text NOT NULL + , warehouse_supervisor text UNIQUE + , warehouse_description text + , CHECK (upper(warehouse_manager) != upper(warehouse_supervisor)) + ) + CREATE TABLE inventory + ( warehouse_id integer REFERENCES warehouse + ON UPDATE CASCADE + ON DELETE RESTRICT + , product_id integer REFERENCES product.product + ON UPDATE CASCADE + ON DELETE RESTRICT + , PRIMARY KEY(warehouse_id, product_id) + , quantity integer NOT NULL + CHECK(quantity > 0) + ) + CREATE VIEW products AS + SELECT DISTINCT product.* + FROM inventory + JOIN product.product USING (product_id); + +-- Sample index +CREATE INDEX quantity_index ON warehouse.inventory (quantity); + +-- +-- Simple text comments +-- +--COMMENT ON DATABASE IS +--'This database has been created for the purpose of simple +-- tests on PostgreSQL Autodoc.'; + +COMMENT ON SCHEMA product IS +'This schema stores a list of products and information + about the product'; + +COMMENT ON SCHEMA warehouse IS +'A list of warehouses and information on warehouses'; + +COMMENT ON TABLE warehouse.inventory IS +'Warehouse inventory'; + +COMMENT ON TABLE store.inventory IS +'Store inventory'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_code IS +'Internal code which represents warehouses for + invoice purposes'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_supervisor IS +'Supervisors name for a warehouse when one + has been assigned. The same supervisor may not + be assigned to more than one warehouse, per company + policy XYZ.'; + +COMMENT ON COLUMN warehouse.warehouse.warehouse_manager IS +'Name of Warehouse Manager'; + +-- +-- A few simple functions +-- +CREATE FUNCTION product.worker(integer, integer) RETURNS integer AS +'SELECT $1 + $1;' LANGUAGE sql; + +CREATE FUNCTION warehouse.worker(integer, integer) RETURNS integer AS +'SELECT $1 * $1;' LANGUAGE sql; + +COMMENT ON FUNCTION product.worker(integer, integer) IS +'Worker function appropriate for products'; + +COMMENT ON FUNCTION warehouse.worker(integer, integer) IS +'Worker function appropriate for warehouses.'; + + +-- +-- Inheritance +-- +CREATE SCHEMA inherit + CREATE TABLE taba (cola integer) + CREATE TABLE tabb (colb integer) inherits(taba) + CREATE TABLE tab1 (col1 integer) + CREATE TABLE tab1b (col1b integer) inherits(tab1, tabb); + +COMMIT; diff --git a/tests/functional/film-schema-0.1.sql b/tests/functional/film-schema-0.1.sql new file mode 100644 index 0000000..11ffa4c --- /dev/null +++ b/tests/functional/film-schema-0.1.sql @@ -0,0 +1,5 @@ +CREATE TABLE film ( + id INTEGER NOT NULL PRIMARY KEY, + title VARCHAR(32) NOT NULL, + release_year INTEGER NOT NULL CHECK (release_year >= 1888) +); diff --git a/tests/functional/film-schema-0.2.sql b/tests/functional/film-schema-0.2.sql new file mode 100644 index 0000000..96efc13 --- /dev/null +++ b/tests/functional/film-schema-0.2.sql @@ -0,0 +1,3 @@ +ALTER TABLE film ADD COLUMN length SMALLINT + CHECK (length > 0 AND length < 10000); +ALTER TABLE film ADD COLUMN rating CHAR(5); diff --git a/tests/functional/film-schema-0.3a.sql b/tests/functional/film-schema-0.3a.sql new file mode 100644 index 0000000..90cf545 --- /dev/null +++ b/tests/functional/film-schema-0.3a.sql @@ -0,0 +1,8 @@ +CREATE TABLE language ( + language_id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(20) NOT NULL, + last_update TIMESTAMP WITH TIME ZONE NOT NULL +); +ALTER TABLE film ADD COLUMN language_id INTEGER NOT NULL; +ALTER TABLE film ADD FOREIGN KEY (language_id) + REFERENCES language (language_id); diff --git a/tests/functional/film-schema-0.3b.sql b/tests/functional/film-schema-0.3b.sql new file mode 100644 index 0000000..e02dfc3 --- /dev/null +++ b/tests/functional/film-schema-0.3b.sql @@ -0,0 +1,34 @@ +CREATE TABLE genre ( + genre_id INTEGER PRIMARY KEY, + name VARCHAR(25) NOT NULL +); +INSERT INTO genre VALUES +(1, 'Action'), +(2, 'Adventure'), +(3, 'Animation'), +(4, 'Biography'), +(5, 'Comedy'), +(6, 'Crime'), +(7, 'Documentary'), +(8, 'Drama'), +(9, 'Family'), +(10, 'Fantasy'), +(11, 'Film-Noir'), +(12, 'History'), +(13, 'Horror'), +(14, 'Music'), +(15, 'Musical'), +(16, 'Mystery'), +(17, 'Romance'), +(18, 'Sci-Fi'), +(19, 'Sport'), +(20, 'Thriller'), +(21, 'War'), +(22, 'Western'); +CREATE TABLE film_genre ( + film_id INTEGER NOT NULL REFERENCES film (id), + genre_id INTEGER NOT NULL + REFERENCES genre (genre_id), + last_update TIMESTAMP WITH TIME ZONE NOT NULL, + PRIMARY KEY (film_id, genre_id) +); diff --git a/tests/functional/pagila-schema.sql b/tests/functional/pagila-schema.sql new file mode 100644 index 0000000..82fdf81 --- /dev/null +++ b/tests/functional/pagila-schema.sql @@ -0,0 +1,1748 @@ +-- +-- PostgreSQL database dump +-- + +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = off; +SET check_function_bodies = false; +SET client_min_messages = warning; +SET escape_string_warning = off; + +-- +-- Name: plpgsql; Type: PROCEDURAL LANGUAGE; Schema: -; Owner: postgres +-- + +-- CREATE PROCEDURAL LANGUAGE plpgsql; + + +-- ALTER PROCEDURAL LANGUAGE plpgsql OWNER TO postgres; + +CREATE SCHEMA IF NOT EXISTS public AUTHORIZATION postgres; + +SET search_path = public, pg_catalog; + +-- +-- Name: actor_actor_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE actor_actor_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.actor_actor_id_seq OWNER TO postgres; + +SET default_tablespace = ''; + +SET default_with_oids = false; + +-- +-- Name: actor; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE actor ( + actor_id integer DEFAULT nextval('actor_actor_id_seq'::regclass) NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.actor OWNER TO postgres; + +-- +-- Name: mpaa_rating; Type: TYPE; Schema: public; Owner: postgres +-- + +CREATE TYPE mpaa_rating AS ENUM ( + 'G', + 'PG', + 'PG-13', + 'R', + 'NC-17' +); + + +ALTER TYPE public.mpaa_rating OWNER TO postgres; + +-- +-- Name: year; Type: DOMAIN; Schema: public; Owner: postgres +-- + +CREATE DOMAIN year AS integer + CONSTRAINT year_check CHECK (((VALUE >= 1901) AND (VALUE <= 2155))); + + +ALTER DOMAIN public.year OWNER TO postgres; + +-- +-- Name: _group_concat(text, text); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION _group_concat(text, text) RETURNS text + AS $_$ +SELECT CASE + WHEN $2 IS NULL THEN $1 + WHEN $1 IS NULL THEN $2 + ELSE $1 || ', ' || $2 +END +$_$ + LANGUAGE sql IMMUTABLE; + + +ALTER FUNCTION public._group_concat(text, text) OWNER TO postgres; + +-- +-- Name: group_concat(text); Type: AGGREGATE; Schema: public; Owner: postgres +-- + +CREATE AGGREGATE group_concat(text) ( + SFUNC = _group_concat, + STYPE = text +); + + +ALTER AGGREGATE public.group_concat(text) OWNER TO postgres; + +-- +-- Name: category_category_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE category_category_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.category_category_id_seq OWNER TO postgres; + +-- +-- Name: category; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE category ( + category_id integer DEFAULT nextval('category_category_id_seq'::regclass) NOT NULL, + name character varying(25) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.category OWNER TO postgres; + +-- +-- Name: film_film_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE film_film_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.film_film_id_seq OWNER TO postgres; + +-- +-- Name: film; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film ( + film_id integer DEFAULT nextval('film_film_id_seq'::regclass) NOT NULL, + title character varying(255) NOT NULL, + description text, + release_year year, + language_id smallint NOT NULL, + original_language_id smallint, + rental_duration smallint DEFAULT 3 NOT NULL, + rental_rate numeric(4,2) DEFAULT 4.99 NOT NULL, + length smallint, + replacement_cost numeric(5,2) DEFAULT 19.99 NOT NULL, + rating mpaa_rating DEFAULT 'G'::mpaa_rating, + last_update timestamp without time zone DEFAULT now() NOT NULL, + special_features text[], + fulltext tsvector NOT NULL +); + + +ALTER TABLE public.film OWNER TO postgres; + +-- +-- Name: film_actor; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film_actor ( + actor_id smallint NOT NULL, + film_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.film_actor OWNER TO postgres; + +-- +-- Name: film_category; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE film_category ( + film_id smallint NOT NULL, + category_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.film_category OWNER TO postgres; + +-- +-- Name: actor_info; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW actor_info AS + SELECT a.actor_id, a.first_name, a.last_name, group_concat(DISTINCT (((c.name)::text || ': '::text) || (SELECT group_concat((f.title)::text) AS group_concat FROM ((film f JOIN film_category fc ON ((f.film_id = fc.film_id))) JOIN film_actor fa ON ((f.film_id = fa.film_id))) WHERE ((fc.category_id = c.category_id) AND (fa.actor_id = a.actor_id)) GROUP BY fa.actor_id))) AS film_info FROM (((actor a LEFT JOIN film_actor fa ON ((a.actor_id = fa.actor_id))) LEFT JOIN film_category fc ON ((fa.film_id = fc.film_id))) LEFT JOIN category c ON ((fc.category_id = c.category_id))) GROUP BY a.actor_id, a.first_name, a.last_name; + + +ALTER TABLE public.actor_info OWNER TO postgres; + +-- +-- Name: address_address_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE address_address_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.address_address_id_seq OWNER TO postgres; + +-- +-- Name: address; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE address ( + address_id integer DEFAULT nextval('address_address_id_seq'::regclass) NOT NULL, + address character varying(50) NOT NULL, + address2 character varying(50), + district character varying(20) NOT NULL, + city_id smallint NOT NULL, + postal_code character varying(10), + phone character varying(20) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.address OWNER TO postgres; + +-- +-- Name: city_city_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE city_city_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.city_city_id_seq OWNER TO postgres; + +-- +-- Name: city; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE city ( + city_id integer DEFAULT nextval('city_city_id_seq'::regclass) NOT NULL, + city character varying(50) NOT NULL, + country_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.city OWNER TO postgres; + +-- +-- Name: country_country_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE country_country_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.country_country_id_seq OWNER TO postgres; + +-- +-- Name: country; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE country ( + country_id integer DEFAULT nextval('country_country_id_seq'::regclass) NOT NULL, + country character varying(50) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.country OWNER TO postgres; + +-- +-- Name: customer_customer_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE customer_customer_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.customer_customer_id_seq OWNER TO postgres; + +-- +-- Name: customer; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE customer ( + customer_id integer DEFAULT nextval('customer_customer_id_seq'::regclass) NOT NULL, + store_id smallint NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + email character varying(50), + address_id smallint NOT NULL, + activebool boolean DEFAULT true NOT NULL, + create_date date DEFAULT ('now'::text)::date NOT NULL, + last_update timestamp without time zone DEFAULT now(), + active integer +); + + +ALTER TABLE public.customer OWNER TO postgres; + +-- +-- Name: customer_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW customer_list AS + SELECT cu.customer_id AS id, (((cu.first_name)::text || ' '::text) || (cu.last_name)::text) AS name, a.address, a.postal_code AS "zip code", a.phone, city.city, country.country, CASE WHEN cu.activebool THEN 'active'::text ELSE ''::text END AS notes, cu.store_id AS sid FROM (((customer cu JOIN address a ON ((cu.address_id = a.address_id))) JOIN city ON ((a.city_id = city.city_id))) JOIN country ON ((city.country_id = country.country_id))); + + +ALTER TABLE public.customer_list OWNER TO postgres; + +-- +-- Name: film_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW film_list AS + SELECT film.film_id AS fid, film.title, film.description, category.name AS category, film.rental_rate AS price, film.length, film.rating, group_concat((((actor.first_name)::text || ' '::text) || (actor.last_name)::text)) AS actors FROM ((((category LEFT JOIN film_category ON ((category.category_id = film_category.category_id))) LEFT JOIN film ON ((film_category.film_id = film.film_id))) JOIN film_actor ON ((film.film_id = film_actor.film_id))) JOIN actor ON ((film_actor.actor_id = actor.actor_id))) GROUP BY film.film_id, film.title, film.description, category.name, film.rental_rate, film.length, film.rating; + + +ALTER TABLE public.film_list OWNER TO postgres; + +-- +-- Name: inventory_inventory_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE inventory_inventory_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.inventory_inventory_id_seq OWNER TO postgres; + +-- +-- Name: inventory; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE inventory ( + inventory_id integer DEFAULT nextval('inventory_inventory_id_seq'::regclass) NOT NULL, + film_id smallint NOT NULL, + store_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.inventory OWNER TO postgres; + +-- +-- Name: language_language_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE language_language_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.language_language_id_seq OWNER TO postgres; + +-- +-- Name: language; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE language ( + language_id integer DEFAULT nextval('language_language_id_seq'::regclass) NOT NULL, + name character(20) NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.language OWNER TO postgres; + +-- +-- Name: nicer_but_slower_film_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW nicer_but_slower_film_list AS + SELECT film.film_id AS fid, film.title, film.description, category.name AS category, film.rental_rate AS price, film.length, film.rating, group_concat((((upper("substring"((actor.first_name)::text, 1, 1)) || lower("substring"((actor.first_name)::text, 2))) || upper("substring"((actor.last_name)::text, 1, 1))) || lower("substring"((actor.last_name)::text, 2)))) AS actors FROM ((((category LEFT JOIN film_category ON ((category.category_id = film_category.category_id))) LEFT JOIN film ON ((film_category.film_id = film.film_id))) JOIN film_actor ON ((film.film_id = film_actor.film_id))) JOIN actor ON ((film_actor.actor_id = actor.actor_id))) GROUP BY film.film_id, film.title, film.description, category.name, film.rental_rate, film.length, film.rating; + + +ALTER TABLE public.nicer_but_slower_film_list OWNER TO postgres; + +-- +-- Name: payment_payment_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE payment_payment_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.payment_payment_id_seq OWNER TO postgres; + +-- +-- Name: payment; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment ( + payment_id integer DEFAULT nextval('payment_payment_id_seq'::regclass) NOT NULL, + customer_id smallint NOT NULL, + staff_id smallint NOT NULL, + rental_id integer NOT NULL, + amount numeric(5,2) NOT NULL, + payment_date timestamp without time zone NOT NULL +); + + +ALTER TABLE public.payment OWNER TO postgres; + +-- +-- Name: payment_p2007_01; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_01 (CONSTRAINT payment_p2007_01_payment_date_check CHECK (((payment_date >= '2007-01-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-02-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_01 OWNER TO postgres; + +-- +-- Name: payment_p2007_02; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_02 (CONSTRAINT payment_p2007_02_payment_date_check CHECK (((payment_date >= '2007-02-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-03-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_02 OWNER TO postgres; + +-- +-- Name: payment_p2007_03; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_03 (CONSTRAINT payment_p2007_03_payment_date_check CHECK (((payment_date >= '2007-03-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-04-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_03 OWNER TO postgres; + +-- +-- Name: payment_p2007_04; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_04 (CONSTRAINT payment_p2007_04_payment_date_check CHECK (((payment_date >= '2007-04-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-05-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_04 OWNER TO postgres; + +-- +-- Name: payment_p2007_05; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_05 (CONSTRAINT payment_p2007_05_payment_date_check CHECK (((payment_date >= '2007-05-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-06-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_05 OWNER TO postgres; + +-- +-- Name: payment_p2007_06; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE payment_p2007_06 (CONSTRAINT payment_p2007_06_payment_date_check CHECK (((payment_date >= '2007-06-01 00:00:00'::timestamp without time zone) AND (payment_date < '2007-07-01 00:00:00'::timestamp without time zone))) +) +INHERITS (payment); + + +ALTER TABLE public.payment_p2007_06 OWNER TO postgres; + +-- +-- Name: rental_rental_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE rental_rental_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.rental_rental_id_seq OWNER TO postgres; + +-- +-- Name: rental; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE rental ( + rental_id integer DEFAULT nextval('rental_rental_id_seq'::regclass) NOT NULL, + rental_date timestamp without time zone NOT NULL, + inventory_id integer NOT NULL, + customer_id smallint NOT NULL, + return_date timestamp without time zone, + staff_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.rental OWNER TO postgres; + +-- +-- Name: sales_by_film_category; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW sales_by_film_category AS + SELECT c.name AS category, sum(p.amount) AS total_sales FROM (((((payment p JOIN rental r ON ((p.rental_id = r.rental_id))) JOIN inventory i ON ((r.inventory_id = i.inventory_id))) JOIN film f ON ((i.film_id = f.film_id))) JOIN film_category fc ON ((f.film_id = fc.film_id))) JOIN category c ON ((fc.category_id = c.category_id))) GROUP BY c.name ORDER BY sum(p.amount) DESC; + + +ALTER TABLE public.sales_by_film_category OWNER TO postgres; + +-- +-- Name: staff_staff_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE staff_staff_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.staff_staff_id_seq OWNER TO postgres; + +-- +-- Name: staff; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE staff ( + staff_id integer DEFAULT nextval('staff_staff_id_seq'::regclass) NOT NULL, + first_name character varying(45) NOT NULL, + last_name character varying(45) NOT NULL, + address_id smallint NOT NULL, + email character varying(50), + store_id smallint NOT NULL, + active boolean DEFAULT true NOT NULL, + username character varying(16) NOT NULL, + password character varying(40), + last_update timestamp without time zone DEFAULT now() NOT NULL, + picture bytea +); + + +ALTER TABLE public.staff OWNER TO postgres; + +-- +-- Name: store_store_id_seq; Type: SEQUENCE; Schema: public; Owner: postgres +-- + +CREATE SEQUENCE store_store_id_seq + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER TABLE public.store_store_id_seq OWNER TO postgres; + +-- +-- Name: store; Type: TABLE; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE TABLE store ( + store_id integer DEFAULT nextval('store_store_id_seq'::regclass) NOT NULL, + manager_staff_id smallint NOT NULL, + address_id smallint NOT NULL, + last_update timestamp without time zone DEFAULT now() NOT NULL +); + + +ALTER TABLE public.store OWNER TO postgres; + +-- +-- Name: sales_by_store; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW sales_by_store AS + SELECT (((c.city)::text || ','::text) || (cy.country)::text) AS store, (((m.first_name)::text || ' '::text) || (m.last_name)::text) AS manager, sum(p.amount) AS total_sales FROM (((((((payment p JOIN rental r ON ((p.rental_id = r.rental_id))) JOIN inventory i ON ((r.inventory_id = i.inventory_id))) JOIN store s ON ((i.store_id = s.store_id))) JOIN address a ON ((s.address_id = a.address_id))) JOIN city c ON ((a.city_id = c.city_id))) JOIN country cy ON ((c.country_id = cy.country_id))) JOIN staff m ON ((s.manager_staff_id = m.staff_id))) GROUP BY cy.country, c.city, s.store_id, m.first_name, m.last_name ORDER BY cy.country, c.city; + + +ALTER TABLE public.sales_by_store OWNER TO postgres; + +-- +-- Name: staff_list; Type: VIEW; Schema: public; Owner: postgres +-- + +CREATE VIEW staff_list AS + SELECT s.staff_id AS id, (((s.first_name)::text || ' '::text) || (s.last_name)::text) AS name, a.address, a.postal_code AS "zip code", a.phone, city.city, country.country, s.store_id AS sid FROM (((staff s JOIN address a ON ((s.address_id = a.address_id))) JOIN city ON ((a.city_id = city.city_id))) JOIN country ON ((city.country_id = country.country_id))); + + +ALTER TABLE public.staff_list OWNER TO postgres; + +-- +-- Name: film_in_stock(integer, integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION film_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) RETURNS SETOF integer + AS $_$ + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND inventory_in_stock(inventory_id); +$_$ + LANGUAGE sql; + + +ALTER FUNCTION public.film_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) OWNER TO postgres; + +-- +-- Name: film_not_in_stock(integer, integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION film_not_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) RETURNS SETOF integer + AS $_$ + SELECT inventory_id + FROM inventory + WHERE film_id = $1 + AND store_id = $2 + AND NOT inventory_in_stock(inventory_id); +$_$ + LANGUAGE sql; + + +ALTER FUNCTION public.film_not_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer) OWNER TO postgres; + +-- +-- Name: get_customer_balance(integer, timestamp without time zone); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION get_customer_balance(p_customer_id integer, p_effective_date timestamp without time zone) RETURNS numeric + AS $$ + --#OK, WE NEED TO CALCULATE THE CURRENT BALANCE GIVEN A CUSTOMER_ID AND A DATE + --#THAT WE WANT THE BALANCE TO BE EFFECTIVE FOR. THE BALANCE IS: + --# 1) RENTAL FEES FOR ALL PREVIOUS RENTALS + --# 2) ONE DOLLAR FOR EVERY DAY THE PREVIOUS RENTALS ARE OVERDUE + --# 3) IF A FILM IS MORE THAN RENTAL_DURATION * 2 OVERDUE, CHARGE THE REPLACEMENT_COST + --# 4) SUBTRACT ALL PAYMENTS MADE BEFORE THE DATE SPECIFIED +DECLARE + v_rentfees DECIMAL(5,2); --#FEES PAID TO RENT THE VIDEOS INITIALLY + v_overfees INTEGER; --#LATE FEES FOR PRIOR RENTALS + v_payments DECIMAL(5,2); --#SUM OF PAYMENTS MADE PREVIOUSLY +BEGIN + SELECT COALESCE(SUM(film.rental_rate),0) INTO v_rentfees + FROM film, inventory, rental + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + SELECT COALESCE(SUM(IF((rental.return_date - rental.rental_date) > (film.rental_duration * '1 day'::interval), + ((rental.return_date - rental.rental_date) - (film.rental_duration * '1 day'::interval)),0)),0) INTO v_overfees + FROM rental, inventory, film + WHERE film.film_id = inventory.film_id + AND inventory.inventory_id = rental.inventory_id + AND rental.rental_date <= p_effective_date + AND rental.customer_id = p_customer_id; + + SELECT COALESCE(SUM(payment.amount),0) INTO v_payments + FROM payment + WHERE payment.payment_date <= p_effective_date + AND payment.customer_id = p_customer_id; + + RETURN v_rentfees + v_overfees - v_payments; +END +$$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.get_customer_balance(p_customer_id integer, p_effective_date timestamp without time zone) OWNER TO postgres; + +-- +-- Name: inventory_held_by_customer(integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION inventory_held_by_customer(p_inventory_id integer) RETURNS integer + AS $$ +DECLARE + v_customer_id INTEGER; +BEGIN + + SELECT customer_id INTO v_customer_id + FROM rental + WHERE return_date IS NULL + AND inventory_id = p_inventory_id; + + RETURN v_customer_id; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.inventory_held_by_customer(p_inventory_id integer) OWNER TO postgres; + +-- +-- Name: inventory_in_stock(integer); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION inventory_in_stock(p_inventory_id integer) RETURNS boolean + AS $$ +DECLARE + v_rentals INTEGER; + v_out INTEGER; +BEGIN + -- AN ITEM IS IN-STOCK IF THERE ARE EITHER NO ROWS IN THE rental TABLE + -- FOR THE ITEM OR ALL ROWS HAVE return_date POPULATED + + SELECT count(*) INTO v_rentals + FROM rental + WHERE inventory_id = p_inventory_id; + + IF v_rentals = 0 THEN + RETURN TRUE; + END IF; + + SELECT COUNT(rental_id) INTO v_out + FROM inventory LEFT JOIN rental USING(inventory_id) + WHERE inventory.inventory_id = p_inventory_id + AND rental.return_date IS NULL; + + IF v_out > 0 THEN + RETURN FALSE; + ELSE + RETURN TRUE; + END IF; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.inventory_in_stock(p_inventory_id integer) OWNER TO postgres; + +-- +-- Name: last_day(timestamp without time zone); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION last_day(timestamp without time zone) RETURNS date + AS $_$ + SELECT CASE + WHEN EXTRACT(MONTH FROM $1) = 12 THEN + (((EXTRACT(YEAR FROM $1) + 1) operator(pg_catalog.||) '-01-01')::date - INTERVAL '1 day')::date + ELSE + ((EXTRACT(YEAR FROM $1) operator(pg_catalog.||) '-' operator(pg_catalog.||) (EXTRACT(MONTH FROM $1) + 1) operator(pg_catalog.||) '-01')::date - INTERVAL '1 day')::date + END +$_$ + LANGUAGE sql IMMUTABLE STRICT; + + +ALTER FUNCTION public.last_day(timestamp without time zone) OWNER TO postgres; + +-- +-- Name: last_updated(); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION last_updated() RETURNS trigger + AS $$ +BEGIN + NEW.last_update = CURRENT_TIMESTAMP; + RETURN NEW; +END $$ + LANGUAGE plpgsql; + + +ALTER FUNCTION public.last_updated() OWNER TO postgres; + +-- +-- Name: rewards_report(integer, numeric); Type: FUNCTION; Schema: public; Owner: postgres +-- + +CREATE FUNCTION rewards_report(min_monthly_purchases integer, min_dollar_amount_purchased numeric) RETURNS SETOF customer + AS $_$ +DECLARE + last_month_start DATE; + last_month_end DATE; +rr RECORD; +tmpSQL TEXT; +BEGIN + + /* Some sanity checks... */ + IF min_monthly_purchases = 0 THEN + RAISE EXCEPTION 'Minimum monthly purchases parameter must be > 0'; + END IF; + IF min_dollar_amount_purchased = 0.00 THEN + RAISE EXCEPTION 'Minimum monthly dollar amount purchased parameter must be > $0.00'; + END IF; + + last_month_start := CURRENT_DATE - '3 month'::interval; + last_month_start := to_date((extract(YEAR FROM last_month_start) || '-' || extract(MONTH FROM last_month_start) || '-01'),'YYYY-MM-DD'); + last_month_end := LAST_DAY(last_month_start); + + /* + Create a temporary storage area for Customer IDs. + */ + CREATE TEMPORARY TABLE tmpCustomer (customer_id INTEGER NOT NULL PRIMARY KEY); + + /* + Find all customers meeting the monthly purchase requirements + */ + + tmpSQL := 'INSERT INTO tmpCustomer (customer_id) + SELECT p.customer_id + FROM payment AS p + WHERE DATE(p.payment_date) BETWEEN '||quote_literal(last_month_start) ||' AND '|| quote_literal(last_month_end) || ' + GROUP BY customer_id + HAVING SUM(p.amount) > '|| min_dollar_amount_purchased || ' + AND COUNT(customer_id) > ' ||min_monthly_purchases ; + + EXECUTE tmpSQL; + + /* + Output ALL customer information of matching rewardees. + Customize output as needed. + */ + FOR rr IN EXECUTE 'SELECT c.* FROM tmpCustomer AS t INNER JOIN customer AS c ON t.customer_id = c.customer_id' LOOP + RETURN NEXT rr; + END LOOP; + + /* Clean up */ + tmpSQL := 'DROP TABLE tmpCustomer'; + EXECUTE tmpSQL; + +RETURN; +END +$_$ + LANGUAGE plpgsql SECURITY DEFINER; + + +ALTER FUNCTION public.rewards_report(min_monthly_purchases integer, min_dollar_amount_purchased numeric) OWNER TO postgres; + +-- +-- Name: actor_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY actor + ADD CONSTRAINT actor_pkey PRIMARY KEY (actor_id); + + +-- +-- Name: address_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY address + ADD CONSTRAINT address_pkey PRIMARY KEY (address_id); + + +-- +-- Name: category_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY category + ADD CONSTRAINT category_pkey PRIMARY KEY (category_id); + + +-- +-- Name: city_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY city + ADD CONSTRAINT city_pkey PRIMARY KEY (city_id); + + +-- +-- Name: country_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY country + ADD CONSTRAINT country_pkey PRIMARY KEY (country_id); + + +-- +-- Name: customer_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_pkey PRIMARY KEY (customer_id); + + +-- +-- Name: film_actor_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_pkey PRIMARY KEY (actor_id, film_id); + + +-- +-- Name: film_category_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_pkey PRIMARY KEY (film_id, category_id); + + +-- +-- Name: film_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_pkey PRIMARY KEY (film_id); + + +-- +-- Name: inventory_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_pkey PRIMARY KEY (inventory_id); + + +-- +-- Name: language_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY language + ADD CONSTRAINT language_pkey PRIMARY KEY (language_id); + + +-- +-- Name: payment_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_pkey PRIMARY KEY (payment_id); + + +-- +-- Name: rental_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_pkey PRIMARY KEY (rental_id); + + +-- +-- Name: staff_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_pkey PRIMARY KEY (staff_id); + + +-- +-- Name: store_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres; Tablespace: +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_pkey PRIMARY KEY (store_id); + + +-- +-- Name: film_fulltext_idx; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX film_fulltext_idx ON film USING gist (fulltext); + + +-- +-- Name: idx_actor_last_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_actor_last_name ON actor USING btree (last_name); + + +-- +-- Name: idx_fk_address_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_address_id ON customer USING btree (address_id); + + +-- +-- Name: idx_fk_city_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_city_id ON address USING btree (city_id); + + +-- +-- Name: idx_fk_country_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_country_id ON city USING btree (country_id); + + +-- +-- Name: idx_fk_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_customer_id ON payment USING btree (customer_id); + + +-- +-- Name: idx_fk_film_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_film_id ON film_actor USING btree (film_id); + + +-- +-- Name: idx_fk_inventory_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_inventory_id ON rental USING btree (inventory_id); + + +-- +-- Name: idx_fk_language_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_language_id ON film USING btree (language_id); + + +-- +-- Name: idx_fk_original_language_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_original_language_id ON film USING btree (original_language_id); + + +-- +-- Name: idx_fk_payment_p2007_01_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_01_customer_id ON payment_p2007_01 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_01_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_01_staff_id ON payment_p2007_01 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_02_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_02_customer_id ON payment_p2007_02 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_02_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_02_staff_id ON payment_p2007_02 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_03_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_03_customer_id ON payment_p2007_03 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_03_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_03_staff_id ON payment_p2007_03 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_04_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_04_customer_id ON payment_p2007_04 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_04_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_04_staff_id ON payment_p2007_04 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_05_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_05_customer_id ON payment_p2007_05 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_05_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_05_staff_id ON payment_p2007_05 USING btree (staff_id); + + +-- +-- Name: idx_fk_payment_p2007_06_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_06_customer_id ON payment_p2007_06 USING btree (customer_id); + + +-- +-- Name: idx_fk_payment_p2007_06_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_payment_p2007_06_staff_id ON payment_p2007_06 USING btree (staff_id); + + +-- +-- Name: idx_fk_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_staff_id ON payment USING btree (staff_id); + + +-- +-- Name: idx_fk_store_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_fk_store_id ON customer USING btree (store_id); + + +-- +-- Name: idx_last_name; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_last_name ON customer USING btree (last_name); + + +-- +-- Name: idx_store_id_film_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_store_id_film_id ON inventory USING btree (store_id, film_id); + + +-- +-- Name: idx_title; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE INDEX idx_title ON film USING btree (title); + + +-- +-- Name: idx_unq_manager_staff_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX idx_unq_manager_staff_id ON store USING btree (manager_staff_id); + + +-- +-- Name: idx_unq_rental_rental_date_inventory_id_customer_id; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX idx_unq_rental_rental_date_inventory_id_customer_id ON rental USING btree (rental_date, inventory_id, customer_id); + + +-- +-- Name: payment_p2007_01_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_01_payment_id_unq ON payment_p2007_01 USING btree (payment_id); + + +-- +-- Name: payment_p2007_02_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_02_payment_id_unq ON payment_p2007_02 USING btree (payment_id); + + +-- +-- Name: payment_p2007_03_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_03_payment_id_unq ON payment_p2007_03 USING btree (payment_id); + + +-- +-- Name: payment_p2007_04_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_04_payment_id_unq ON payment_p2007_04 USING btree (payment_id); + + +-- +-- Name: payment_p2007_05_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_05_payment_id_unq ON payment_p2007_05 USING btree (payment_id); + + +-- +-- Name: payment_p2007_06_payment_id_unq; Type: INDEX; Schema: public; Owner: postgres; Tablespace: +-- + +CREATE UNIQUE INDEX payment_p2007_06_payment_id_unq ON payment_p2007_06 USING btree (payment_id); + + +-- +-- Name: payment_insert_p2007_01; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_01 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-01-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-02-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_01 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_02; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_02 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-02-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-03-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_02 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_03; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_03 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-03-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-04-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_03 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_04; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_04 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-04-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-05-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_04 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_05; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_05 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-05-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-06-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_05 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: payment_insert_p2007_06; Type: RULE; Schema: public; Owner: postgres +-- + +CREATE RULE payment_insert_p2007_06 AS ON INSERT TO payment WHERE ((new.payment_date >= '2007-06-01 00:00:00'::timestamp without time zone) AND (new.payment_date < '2007-07-01 00:00:00'::timestamp without time zone)) DO INSTEAD INSERT INTO payment_p2007_06 (payment_id, customer_id, staff_id, rental_id, amount, payment_date) VALUES (DEFAULT, new.customer_id, new.staff_id, new.rental_id, new.amount, new.payment_date); + + +-- +-- Name: film_fulltext_trigger; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER film_fulltext_trigger + BEFORE INSERT OR UPDATE ON film + FOR EACH ROW + EXECUTE PROCEDURE tsvector_update_trigger('fulltext', 'pg_catalog.english', 'title', 'description'); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON actor + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON address + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON category + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON city + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON country + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON customer + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film_actor + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON film_category + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON inventory + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON language + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON rental + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON staff + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: last_updated; Type: TRIGGER; Schema: public; Owner: postgres +-- + +CREATE TRIGGER last_updated + BEFORE UPDATE ON store + FOR EACH ROW + EXECUTE PROCEDURE last_updated(); + + +-- +-- Name: address_city_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY address + ADD CONSTRAINT address_city_id_fkey FOREIGN KEY (city_id) REFERENCES city(city_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: city_country_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY city + ADD CONSTRAINT city_country_id_fkey FOREIGN KEY (country_id) REFERENCES country(country_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: customer_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: customer_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY customer + ADD CONSTRAINT customer_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_actor_actor_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_actor_id_fkey FOREIGN KEY (actor_id) REFERENCES actor(actor_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_actor_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_actor + ADD CONSTRAINT film_actor_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_category_category_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_category_id_fkey FOREIGN KEY (category_id) REFERENCES category(category_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_category_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film_category + ADD CONSTRAINT film_category_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_language_id_fkey FOREIGN KEY (language_id) REFERENCES language(language_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: film_original_language_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY film + ADD CONSTRAINT film_original_language_id_fkey FOREIGN KEY (original_language_id) REFERENCES language(language_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: inventory_film_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_film_id_fkey FOREIGN KEY (film_id) REFERENCES film(film_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: inventory_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY inventory + ADD CONSTRAINT inventory_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: payment_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: payment_p2007_01_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_01_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_01_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_01 + ADD CONSTRAINT payment_p2007_01_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_02_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_02_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_02_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_02 + ADD CONSTRAINT payment_p2007_02_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_03_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_03_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_03_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_03 + ADD CONSTRAINT payment_p2007_03_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_04_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_04_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_04_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_04 + ADD CONSTRAINT payment_p2007_04_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_05_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_05_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_05_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_05 + ADD CONSTRAINT payment_p2007_05_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_p2007_06_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id); + + +-- +-- Name: payment_p2007_06_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id); + + +-- +-- Name: payment_p2007_06_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment_p2007_06 + ADD CONSTRAINT payment_p2007_06_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id); + + +-- +-- Name: payment_rental_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_rental_id_fkey FOREIGN KEY (rental_id) REFERENCES rental(rental_id) ON UPDATE CASCADE ON DELETE SET NULL; + + +-- +-- Name: payment_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY payment + ADD CONSTRAINT payment_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_customer_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_customer_id_fkey FOREIGN KEY (customer_id) REFERENCES customer(customer_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_inventory_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_inventory_id_fkey FOREIGN KEY (inventory_id) REFERENCES inventory(inventory_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: rental_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY rental + ADD CONSTRAINT rental_staff_id_fkey FOREIGN KEY (staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: staff_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: staff_store_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY staff + ADD CONSTRAINT staff_store_id_fkey FOREIGN KEY (store_id) REFERENCES store(store_id); + + +-- +-- Name: store_address_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_address_id_fkey FOREIGN KEY (address_id) REFERENCES address(address_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: store_manager_staff_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: postgres +-- + +ALTER TABLE ONLY store + ADD CONSTRAINT store_manager_staff_id_fkey FOREIGN KEY (manager_staff_id) REFERENCES staff(staff_id) ON UPDATE CASCADE ON DELETE RESTRICT; + + +-- +-- Name: public; Type: ACL; Schema: -; Owner: postgres +-- + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO postgres; +GRANT ALL ON SCHEMA public TO PUBLIC; + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/tests/functional/test_autodoc.py b/tests/functional/test_autodoc.py new file mode 100644 index 0000000..374324c --- /dev/null +++ b/tests/functional/test_autodoc.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using autodoc schema + +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/~checkout~/autodoc/autodoc/ +regressdatabase.sql?rev=1.2 +""" +from pyrseas.testutils import DbMigrateTestCase + + +class AutodocTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('autodoc') + cls.remove_tempfiles('empty') + + def test_autodoc(self): + # Create the source schema + self.execute_script(__file__, 'autodoc-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('autodoc-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + emptydump = self.tempfile_path('empty.dump') + self.run_pg_dump(emptydump) + emptyyaml = self.tempfile_path('empty.yaml') + self.create_yaml(emptyyaml) + + # Migrate the target database + targsql = self.tempfile_path('autodoc.sql') + self.migrate_target(srcyaml, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc.yaml') + self.create_yaml(targyaml) + + # diff autodoc-src.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff autodoc-src.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.migrate_target(emptyyaml, targsql) + + # Run pg_dump against target database + self.run_pg_dump(targdump) + + # Create target YAML file + self.create_yaml(targyaml) + + # diff empty.dump against autodoc.dump + assert self.lines(emptydump) == self.lines(targdump) + # diff empty.yaml against autodoc.yaml + assert self.lines(emptyyaml) == self.lines(targyaml) diff --git a/tests/functional/test_autodoc_dir.py b/tests/functional/test_autodoc_dir.py new file mode 100644 index 0000000..92b9d79 --- /dev/null +++ b/tests/functional/test_autodoc_dir.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using autodoc schema but I/O to/from a directory + +Same as test_autodoc.py but with directory tree instead of a single YAML file. +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/~checkout~/autodoc/autodoc/ +regressdatabase.sql?rev=1.2 +""" +from pyrseas.testutils import DbMigrateTestCase + + +class AutodocTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + self.remove_tempfiles('metadata') + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('autodoc') + cls.remove_tempfiles('metadata') + + def test_autodoc(self): + # Create the source schema + self.execute_script(__file__, 'autodoc-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file and directory tree + # Note: the single YAML file is for verification against the target, + # the YAML directory tree is for processing + srcyaml = self.tempfile_path('autodoc-src.yaml') + self.create_yaml(srcyaml, True) + self.create_yaml(None, True) + + # Migrate the target database + targsql = self.tempfile_path('autodoc.sql') + self.migrate_target(None, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc.yaml') + self.create_yaml(targyaml) + + # diff autodoc-src.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff autodoc-src.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.srcdb.execute_commit( + "DROP SCHEMA inherit, product, store, warehouse CASCADE") + # Create source YAML file and directory tree + srcyaml = self.tempfile_path('autodoc-src-empty.yaml') + self.create_yaml(srcyaml, True) + self.create_yaml(None, True) + targsql = self.tempfile_path('autodoc-empty.sql') + self.migrate_target(None, targsql) + + # Run pg_dump against source database + srcdump = self.tempfile_path('autodoc-src-empty.dump') + self.run_pg_dump(srcdump, True) + + # Run pg_dump against target database + targdump = self.tempfile_path('autodoc-empty.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('autodoc-empty.yaml') + self.create_yaml(targyaml) + + # diff empty.dump against autodoc.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff empty.yaml against autodoc.yaml + assert self.lines(srcyaml) == self.lines(targyaml) diff --git a/tests/functional/test_filmversions.py b/tests/functional/test_filmversions.py new file mode 100644 index 0000000..d51a55f --- /dev/null +++ b/tests/functional/test_filmversions.py @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using film versions schema + +See https://pyrseas.wordpress.com/2011/02/07/ +version-control-part-2-sql-databases/ +""" +import os + +from pyrseas.testutils import DbMigrateTestCase +from pyrseas.yamlutil import yamldump + + +class FilmTestCase(DbMigrateTestCase): + + def setUp(self): + self.remove_public_schema(self.srcdb) + + @classmethod + def tearDownClass(cls): + cls.remove_tempfiles('film-0.') + cls.remove_tempfiles('usercfg.yaml') + cls.remove_tempfiles('config.yaml') + cls.remove_tempfiles('metadata') + + def test_film_version_01(self): + "Create schema version 0.1" + self.execute_script(__file__, 'film-schema-0.1.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.1-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('film-0.1-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + self.run_pg_dump(self.tempfile_path('film-0.0.dump')) + self.create_yaml(self.tempfile_path('film-0.0.yaml')) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.1.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.1.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.1.yaml') + self.create_yaml(targyaml) + + # diff film-0.1-src.dump against film-0.1.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.1-src.yaml against film-0.1.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_02(self): + "Update schema to version 0.2" + self.execute_script(__file__, 'film-schema-0.2.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.2-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('film-0.2-src.yaml') + self.create_yaml(srcyaml, True) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.2.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.2.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.2.yaml') + self.create_yaml(targyaml) + + # diff film-0.2-src.dump against film-0.2.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.2-src.yaml against film-0.2.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_03(self): + "Update schema to version 0.3" + self.execute_script(__file__, 'film-schema-0.3a.sql') + self.execute_script(__file__, 'film-schema-0.3b.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('film-0.3-src.dump') + self.run_pg_dump(srcdump, True, True) + + # Create source YAML file + usercfg = self.tempfile_path("usercfg.yaml") + with open(usercfg, 'w') as f: + f.write(yamldump({'repository': {'path': self.tempfile_path('')}})) + os.environ["PYRSEAS_USER_CONFIG"] = usercfg + with open(self.tempfile_path("config.yaml"), 'w') as f: + f.write(yamldump({'datacopy': {'schema sd': ['genre']}})) + srcyaml = self.tempfile_path('film-0.3-src.yaml') + self.create_yaml(srcyaml, True) + + # Migrate the target database + self.migrate_target(srcyaml, self.tempfile_path('film-0.3.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.3.dump') + self.run_pg_dump(targdump, False, True) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.3.yaml') + self.create_yaml(targyaml) + + # diff film-0.3-src.dump against film-0.3.dump + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.3-src.yaml against film-0.3.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_04(self): + "Revert to schema version 0.2" + srcyaml = self.tempfile_path('film-0.2.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.3-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.3-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.3-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.2.dump against film-0.3-undo.dump + srcdump = self.tempfile_path('film-0.2.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.2.yaml against film-0.3-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_05(self): + "Revert to schema version 0.1" + srcyaml = self.tempfile_path('film-0.1.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.2-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.2-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.2-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.1.dump against film-0.2-undo.dump + srcdump = self.tempfile_path('film-0.1.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.1.yaml against film-0.2-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + def test_film_version_06(self): + "Revert to empty schema" + srcyaml = self.tempfile_path('film-0.0.yaml') + self.migrate_target(srcyaml, self.tempfile_path('film-0.1-undo.sql')) + + # Run pg_dump against target database + targdump = self.tempfile_path('film-0.1-undo.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('film-0.1-undo.yaml') + self.create_yaml(targyaml) + + # diff film-0.0.dump against film-0.1-undo.dump + srcdump = self.tempfile_path('film-0.0.dump') + assert self.lines(srcdump) == self.lines(targdump) + # diff film-0.0.yaml against film-0.1-undo.yaml + assert self.lines(srcyaml) == self.lines(targyaml) diff --git a/tests/functional/test_pagila.py b/tests/functional/test_pagila.py new file mode 100644 index 0000000..984269b --- /dev/null +++ b/tests/functional/test_pagila.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +"""Test dbtoyaml and yamltodb using pagila schema + +See http://cvs.pgfoundry.org/cgi-bin/cvsweb.cgi/dbsamples/pagila/ +pagila-schema.sql?rev=1.8 +""" +from difflib import unified_diff + +from pyrseas.testutils import DbMigrateTestCase + + +class PagilaTestCase(DbMigrateTestCase): + + def setUp(self): + super(DbMigrateTestCase, self).setUp() + self.add_public_schema(self.srcdb) + self.add_public_schema(self.db) + + @classmethod + def tearDown(cls): + cls.remove_tempfiles('pagila') + cls.remove_tempfiles('empty') + + def test_pagila(self): + # Create the source schema + self.execute_script(__file__, 'pagila-schema.sql') + + # Run pg_dump against source database + srcdump = self.tempfile_path('pagila-src.dump') + self.run_pg_dump(srcdump, True) + + # Create source YAML file + srcyaml = self.tempfile_path('pagila-src.yaml') + self.create_yaml(srcyaml, True) + + # Run pg_dump/dbtoyaml against empty target database + emptydump = self.tempfile_path('empty.dump') + self.run_pg_dump(emptydump) + emptyyaml = self.tempfile_path('empty.yaml') + self.create_yaml(emptyyaml) + + # Migrate the target database + targsql = self.tempfile_path('pagila.sql') + self.migrate_target(srcyaml, targsql) + + # Run pg_dump against target database + targdump = self.tempfile_path('pagila.dump') + self.run_pg_dump(targdump) + + # Create target YAML file + targyaml = self.tempfile_path('pagila.yaml') + self.create_yaml(targyaml) + + # diff pagila-src.dump against pagila.dump + # order of triggers requires special handling + adds = [] + subs = [] + for line in unified_diff(self.lines(srcdump), self.lines(targdump)): + if line == '--- \n' or line == '+++ \n' or line.startswith('@@'): + continue + if line[:1] == '+': + adds.append(line[1:-1]) + elif line[:1] == '-': + subs.append(line[1:-1]) + subs = sorted(subs) + for i, line in enumerate(sorted(adds)): + assert line == subs[i] + # diff pagila-src.yaml against pagila.yaml + assert self.lines(srcyaml) == self.lines(targyaml) + + # Undo the changes + self.migrate_target(emptyyaml, targsql) + + # Workaround problem with privileges on schema public + self.db.execute("GRANT ALL ON SCHEMA public TO postgres") + self.db.conn.commit() + # Run pg_dump against target database + self.run_pg_dump(targdump) + + self.db.execute("REVOKE ALL ON SCHEMA public FROM postgres") + self.db.conn.commit() + # Create target YAML file + self.create_yaml(targyaml) + + # diff empty.dump against pagila.dump + assert self.lines(emptydump) == self.lines(targdump) + # diff empty.yaml against pagila.yaml + assert self.lines(emptyyaml) == self.lines(targyaml) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..faab758 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +"""Test configuration files""" + +import os +import sys + +from pyrseas.config import Config +from pyrseas.cmdargs import cmd_parser, parse_args +from pyrseas.yamlutil import yamldump + +USER_CFG_DATA = {'database': {'port': 5433}, + 'output': {'version_comment': True}} +CFG_TABLE_DATA = {'schema public': ['t1', 't2']} +CFG_DATA = {'datacopy': CFG_TABLE_DATA} +CFG_FILE = 'testcfg.yaml' + + +def test_defaults(): + "Create a configuration with defaults" + cfg = Config() + for key in ['audit_columns', 'functions', 'function_templates', 'columns', + 'triggers']: + assert key in cfg['augmenter'] + for key in ['metadata', 'data']: + assert key in cfg['repository'] + + +def test_user_config(tmpdir): + "Test a user configuration file" + f = tmpdir.join(CFG_FILE) + f.write(yamldump(USER_CFG_DATA)) + os.environ["PYRSEAS_USER_CONFIG"] = f.strpath + cfg = Config() + assert cfg['database'] == {'port': 5433} + assert cfg['output'] == {'version_comment': True} + + +def test_repo_config(tmpdir): + "Test a repository configuration file" + ucfg = tmpdir.join(CFG_FILE) + ucfg.write(yamldump({'repository': {'path': tmpdir.strpath}})) + f = tmpdir.join("config.yaml") + f.write(yamldump(CFG_DATA)) + os.environ["PYRSEAS_USER_CONFIG"] = ucfg.strpath + cfg = Config() + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_cmd_parser(tmpdir): + "Test parsing a configuration file specified on the command line" + f = tmpdir.join(CFG_FILE) + f.write(yamldump(CFG_DATA)) + sys.argv = ['testprog', 'testdb', '--config', f.strpath] + os.environ["PYRSEAS_USER_CONFIG"] = '' + parser = cmd_parser("Test description", '0.0.1') + cfg = parse_args(parser) + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_parse_repo_config(tmpdir): + "Test parsing a repository configuration file in the current directory" + f = tmpdir.join('config.yaml') + f.write(yamldump(CFG_DATA)) + os.chdir(tmpdir.strpath) + sys.argv = ['testprog', 'testdb'] + os.environ["PYRSEAS_USER_CONFIG"] = '' + parser = cmd_parser("Test description", '0.0.1') + cfg = parse_args(parser) + assert cfg['datacopy'] == CFG_TABLE_DATA + + +def test_repo_user_config(tmpdir): + "Test a repository path specified in the user config" + usercfg = {'repository': {'path': tmpdir.strpath}} + userf = tmpdir.join("usercfg.yaml") + userf.write(yamldump(usercfg)) + os.environ["PYRSEAS_USER_CONFIG"] = userf.strpath + repof = tmpdir.join("config.yaml") + repof.write(yamldump(CFG_DATA)) + cfg = Config() + assert cfg['datacopy'] == CFG_TABLE_DATA diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..bc91045 --- /dev/null +++ b/tox.ini @@ -0,0 +1,34 @@ +[pg150] +setenv = + PYRSEAS_TEST_PORT={env:PG150_PORT} + +[pg130] +setenv = + PYRSEAS_TEST_PORT={env:PG130_PORT} + +[pg110] +setenv = + PYRSEAS_TEST_PORT={env:PG110_PORT} + +[testenv:py39pg150] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg150]setenv} + +[testenv:py39pg130] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg130]setenv} + +[testenv:py39pg110] +basepython=python3.9 +envdir={toxworkdir}/py39 +setenv = {[pg110]setenv} + +[testenv] +deps=pytest +setenv = + PYTHONPATH = {toxinidir} +passenv = HOME +commands = + py.test tests