diff --git a/_config_base.yml b/_config_base.yml index 2a5d8da35c7..5d287fd64b0 100644 --- a/_config_base.yml +++ b/_config_base.yml @@ -49,12 +49,12 @@ release_info: docker_image: cockroachdb/cockroach build_time: 2020/05/26 11:00:26 (go1.13.4) start_time: 2020-05-26 11:01:26.34274101 +0000 UTC - # v20.2: - # name: v20.2.0 - # version: v20.2.0 - # docker_image: cockroachdb/cockroach-unstable - # build_time: 2020/05/12 11:00:26 (go1.13.4) - # start_time: 2020-05-12 11:01:26.34274101 +0000 UTC + v20.2: + name: v20.2.0 + version: v20.2.0-alpha.1 + docker_image: cockroachdb/cockroach-unstable + build_time: 2020/05/12 11:00:26 (go1.13.4) + start_time: 2020-05-12 11:01:26.34274101 +0000 UTC include: ["_redirects"] diff --git a/_config_cockroachdb.yml b/_config_cockroachdb.yml index b35245a773f..2e60b7b1227 100644 --- a/_config_cockroachdb.yml +++ b/_config_cockroachdb.yml @@ -1,8 +1,6 @@ -homepage_title: CockroachDB Docs - -baseurl: "/docs" +baseurl: /docs destination: _site/docs - +homepage_title: CockroachDB Docs versions: + dev: v20.2 stable: v20.1 - dev: v20.1 diff --git a/_config_cockroachdb_local.yml b/_config_cockroachdb_local.yml index bc4f5c66d8f..746ddaf0d7f 100644 --- a/_config_cockroachdb_local.yml +++ b/_config_cockroachdb_local.yml @@ -4,6 +4,7 @@ exclude: - "v2.0" - "v2.1" - "v19.1" +- "v19.2" - "ci" - "scripts" - "vendor" diff --git a/_data/releases.yml b/_data/releases.yml index 09c85751f2f..a2f4723300f 100644 --- a/_data/releases.yml +++ b/_data/releases.yml @@ -167,6 +167,8 @@ no_windows: true - title: Testing releases releases: + - date: June 2020 + version: v20.2.0-alpha.1 - date: Apr 21, 2020 version: v20.1.0-rc.2 - date: Apr 14, 2020 diff --git a/_includes/sidebar-data-v20.2.json b/_includes/sidebar-data-v20.2.json new file mode 100644 index 00000000000..93af4d4c521 --- /dev/null +++ b/_includes/sidebar-data-v20.2.json @@ -0,0 +1,2181 @@ +[ + { + "title": "Guides", + "is_top_level": true, + "items": [ + { + "title": "Get Started", + "items": [ + { + "title": "Install CockroachDB", + "urls": [ + "/${VERSION}/install-cockroachdb.html", + "/${VERSION}/install-cockroachdb-mac.html", + "/${VERSION}/install-cockroachdb-linux.html", + "/${VERSION}/install-cockroachdb-windows.html" + ] + }, + { + "title": "Start a Local Cluster", + "items": [ + { + "title": "From Binary", + "urls": [ + "/${VERSION}/start-a-local-cluster.html", + "/${VERSION}/secure-a-cluster.html" + ] + }, + { + "title": "In Kubernetes", + "urls": [ + "/${VERSION}/orchestrate-a-local-cluster-with-kubernetes.html", + "/${VERSION}/orchestrate-a-local-cluster-with-kubernetes-insecure.html" + ] + }, + { + "title": "In Docker", + "urls": [ + "/${VERSION}/start-a-local-cluster-in-docker-mac.html", + "/${VERSION}/start-a-local-cluster-in-docker-linux.html", + "/${VERSION}/start-a-local-cluster-in-docker-windows.html" + ] + } + ] + }, + { + "title": "Learn CockroachDB SQL", + "urls": [ + "/${VERSION}/learn-cockroachdb-sql.html" + ] + } + ] + }, + { + "title": "Develop", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/developer-guide-overview.html" + ] + }, + { + "title": "Install a Postgres Client", + "urls": [ + "/${VERSION}/install-client-drivers.html" + ] + }, + { + "title": "Connect to the Database", + "urls": [ + "/${VERSION}/connect-to-the-database.html" + ] + }, + { + "title": "Insert Data", + "urls": [ + "/${VERSION}/insert-data.html" + ] + }, + { + "title": "Query Data", + "urls": [ + "/${VERSION}/query-data.html" + ] + }, + { + "title": "Update Data", + "urls": [ + "/${VERSION}/update-data.html" + ] + }, + { + "title": "Delete Data", + "urls": [ + "/${VERSION}/delete-data.html" + ] + }, + { + "title": "Run Multi-Statement Transactions", + "urls": [ + "/${VERSION}/run-multi-statement-transactions.html" + ] + }, + { + "title": "Error Handling & Troubleshooting", + "urls": [ + "/${VERSION}/error-handling-and-troubleshooting.html" + ] + }, + { + "title": "Make Queries Fast", + "urls": [ + "/${VERSION}/make-queries-fast.html" + ] + } + ] + }, + { + "title": "Deploy", + "items": [ + { + "title": "Production Checklist", + "urls": [ + "/${VERSION}/recommended-production-settings.html" + ] + }, + { + "title": "Topology Patterns", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/topology-patterns.html" + ] + }, + { + "title": "Development", + "urls": [ + "/${VERSION}/topology-development.html" + ] + }, + { + "title": "Basic Production", + "urls": [ + "/${VERSION}/topology-basic-production.html" + ] + }, + { + "title": "Geo-Partitioned Replicas", + "urls": [ + "/${VERSION}/topology-geo-partitioned-replicas.html" + ] + }, + { + "title": "Geo-Partitioned Leaseholders", + "urls": [ + "/${VERSION}/topology-geo-partitioned-leaseholders.html" + ] + }, + { + "title": "Duplicate Indexes", + "urls": [ + "/${VERSION}/topology-duplicate-indexes.html" + ] + }, + { + "title": "Follower Reads", + "urls": [ + "/${VERSION}/topology-follower-reads.html" + ] + }, + { + "title": "Follow-the-Workload", + "urls": [ + "/${VERSION}/topology-follow-the-workload.html" + ] + } + ] + }, + { + "title": "Test Deployment", + "urls": [ + "/${VERSION}/deploy-a-test-cluster.html" + ] + }, + { + "title": "Manual Deployment", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/manual-deployment.html" + ] + }, + { + "title": "On-Premises", + "urls": [ + "/${VERSION}/deploy-cockroachdb-on-premises.html", + "/${VERSION}/deploy-cockroachdb-on-premises-insecure.html" + ] + }, + { + "title": "AWS", + "urls": [ + "/${VERSION}/deploy-cockroachdb-on-aws.html", + "/${VERSION}/deploy-cockroachdb-on-aws-insecure.html" + ] + }, + { + "title": "Azure", + "urls": [ + "/${VERSION}/deploy-cockroachdb-on-microsoft-azure.html", + "/${VERSION}/deploy-cockroachdb-on-microsoft-azure-insecure.html" + ] + }, + { + "title": "Digital Ocean", + "urls": [ + "/${VERSION}/deploy-cockroachdb-on-digital-ocean.html", + "/${VERSION}/deploy-cockroachdb-on-digital-ocean-insecure.html" + ] + }, + { + "title": "Google Cloud Platform GCE", + "urls": [ + "/${VERSION}/deploy-cockroachdb-on-google-cloud-platform.html", + "/${VERSION}/deploy-cockroachdb-on-google-cloud-platform-insecure.html" + ] + } + ] + }, + { + "title": "Orchestrated Deployment", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/orchestration.html" + ] + }, + { + "title": "Kubernetes Single-Cluster Deployment", + "urls": [ + "/${VERSION}/orchestrate-cockroachdb-with-kubernetes.html", + "/${VERSION}/orchestrate-cockroachdb-with-kubernetes-insecure.html" + ] + }, + { + "title": "Kubernetes Multi-Cluster Deployment", + "urls": [ + "/${VERSION}/orchestrate-cockroachdb-with-kubernetes-multi-cluster.html" + ] + }, + { + "title": "Kubernetes Performance Optimization", + "urls": [ + "/${VERSION}/kubernetes-performance.html" + ] + }, + { + "title": "Docker Swarm Deployment", + "urls": [ + "/${VERSION}/orchestrate-cockroachdb-with-docker-swarm.html", + "/${VERSION}/orchestrate-cockroachdb-with-docker-swarm-insecure.html" + ] + }, + { + "title": "Mesosphere DC/OS Deployment", + "urls": [ + "/${VERSION}/orchestrate-cockroachdb-with-mesosphere-insecure.html" + ] + } + ] + }, + { + "title": "Performance", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/performance.html" + ] + }, + { + "title": "Benchmarking Instructions", + "urls": [ + "/${VERSION}/performance-benchmarking-with-tpc-c-10-warehouses.html", + "/${VERSION}/performance-benchmarking-with-tpc-c-1k-warehouses.html", + "/${VERSION}/performance-benchmarking-with-tpc-c-10k-warehouses.html", + "/${VERSION}/performance-benchmarking-with-tpc-c-100k-warehouses.html" + ] + }, + { + "title": "Tuning Best Practices", + "urls": [ + "/${VERSION}/performance-best-practices-overview.html" + ] + }, + { + "title": "Tuning Tutorial", + "urls": [ + "/${VERSION}/performance-tuning.html", + "/${VERSION}/performance-tuning-insecure.html" + ] + } + ] + }, + { + "title": "Security", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/security-overview.html" + ] + }, + { + "title": "Authentication", + "urls": [ + "/${VERSION}/authentication.html" + ] + }, + { + "title": "Encryption", + "urls": [ + "/${VERSION}/encryption.html" + ] + }, + { + "title": "Authorization", + "urls": [ + "/${VERSION}/authorization.html" + ] + }, + { + "title": "SQL Audit Logging", + "urls": [ + "/${VERSION}/sql-audit-logging.html" + ] + }, + { + "title": "GSSAPI Authentication", + "urls": [ + "/${VERSION}/gssapi_authentication.html" + ] + } + ] + }, + { + "title": "Monitoring and Alerting", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/monitoring-and-alerting.html" + ] + }, + { + "title": "Use the Admin UI", + "urls": [ + "/${VERSION}/admin-ui-access-and-navigate.html" + ] + }, + { + "title": "Enable the Node Map", + "urls": [ + "/${VERSION}/enable-node-map.html" + ] + }, + { + "title": "Use Prometheus and Alertmanager", + "urls": [ + "/${VERSION}/monitor-cockroachdb-with-prometheus.html" + ] + } + ] + }, + { + "title": "Replication Controls", + "urls": [ + "/${VERSION}/configure-replication-zones.html" + ] + }, + { + "title": "Enterprise Features", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/enterprise-licensing.html" + ] + }, + { + "title": "Table Partitioning", + "urls": [ + "/${VERSION}/partitioning.html" + ] + }, + { + "title": "Change Data Capture", + "urls": [ + "/${VERSION}/change-data-capture.html" + ] + } + ] + } + ] + }, + { + "title": "Migrate", + "items": [ + { + "title": "Migration Overview", + "urls": [ + "/${VERSION}/migration-overview.html" + ] + }, + { + "title": "Migrate from Oracle", + "urls": [ + "/${VERSION}/migrate-from-oracle.html" + ] + }, + { + "title": "Migrate from Postgres", + "urls": [ + "/${VERSION}/migrate-from-postgres.html" + ] + }, + { + "title": "Migrate from MySQL", + "urls": [ + "/${VERSION}/migrate-from-mysql.html" + ] + }, + { + "title": "Migrate from CSV", + "urls": [ + "/${VERSION}/migrate-from-csv.html" + ] + }, + { + "title": "Migrate from Avro", + "urls": [ + "/${VERSION}/migrate-from-avro.html" + ] + } + ] + }, + { + "title": "Maintain", + "items": [ + { + "title": "Upgrade to CockroachDB v20.2", + "urls": [ + "/${VERSION}/upgrade-cockroach-version.html" + ] + }, + { + "title": "Online Schema Changes", + "urls": [ + "/${VERSION}/online-schema-changes.html" + ] + }, + { + "title": "Manage Long-Running Queries", + "urls": [ + "/${VERSION}/manage-long-running-queries.html" + ] + }, + { + "title": "Decommission Nodes", + "urls": [ + "/${VERSION}/remove-nodes.html" + ] + }, + { + "title": "Back up and Restore Data", + "urls": [ + "/${VERSION}/backup-and-restore.html" + ] + }, + { + "title": "Back up and Restore Data - Advanced Options", + "urls": [ + "/${VERSION}/backup-and-restore-advanced-options.html" + ] + }, + { + "title": "Create a File Server for IMPORT/BACKUP", + "urls": [ + "/${VERSION}/create-a-file-server.html" + ] + }, + { + "title": "Rotate Security Certificates", + "urls": [ + "/${VERSION}/rotate-certificates.html" + ] + } + ] + }, + { + "title": "Troubleshoot", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/troubleshooting-overview.html" + ] + }, + { + "title": "Common Errors", + "urls": [ + "/${VERSION}/common-errors.html" + ] + }, + { + "title": "Troubleshoot Cluster Setup", + "urls": [ + "/${VERSION}/cluster-setup-troubleshooting.html" + ] + }, + { + "title": "Troubleshoot Query Behavior", + "urls": [ + "/${VERSION}/query-behavior-troubleshooting.html" + ] + }, + { + "title": "Understand Debug Logs", + "urls": [ + "/${VERSION}/debug-and-error-logs.html" + ] + }, + { + "title": "Replication Reports", + "urls": [ + "/${VERSION}/query-replication-reports.html" + ] + }, + { + "title": "Support Resources", + "urls": [ + "/${VERSION}/support-resources.html" + ] + }, + { + "title": "File an Issue", + "urls": [ + "/${VERSION}/file-an-issue.html" + ] + } + ] + } + ] + }, + { + "title": "Tutorials", + "is_top_level": true, + "items": [ + { + "title": "Database Features", + "items": [ + { + "title": "Replication & Rebalancing", + "urls": [ + "/${VERSION}/demo-replication-and-rebalancing.html" + ] + }, + { + "title": "Fault Tolerance & Recovery", + "urls": [ + "/${VERSION}/demo-fault-tolerance-and-recovery.html" + ] + }, + { + "title": "Multi-Region Topologies", + "urls": [ + "/${VERSION}/demo-low-latency-multi-region-deployment.html" + ] + }, + { + "title": "Serializable Transactions", + "urls": [ + "/${VERSION}/demo-serializable.html" + ] + }, + { + "title": "Cross-Cloud Migration", + "urls": [ + "/${VERSION}/demo-automatic-cloud-migration.html" + ] + }, + { + "title": "Follow-the-Workload", + "urls": [ + "/${VERSION}/demo-follow-the-workload.html" + ] + }, + { + "title": "Orchestration with Kubernetes", + "urls": [ + "/${VERSION}/orchestration-with-kubernetes.html" + ] + }, + { + "title": "JSON Support", + "urls": [ + "/${VERSION}/demo-json-support.html" + ] + }, + { + "title": "SQL Tuning with EXPLAIN", + "urls": [ + "/${VERSION}/sql-tuning-with-explain.html" + ] + } + ] + }, + { + "title": "Sample Apps", + "items": [ + { + "title": "Hello World", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/hello-world-example-apps.html" + ] + }, + { + "title": "Go", + "urls": [ + "/${VERSION}/build-a-go-app-with-cockroachdb.html", + "/${VERSION}/build-a-go-app-with-cockroachdb-gorm.html" + ] + }, + { + "title": "Python", + "urls": [ + "/${VERSION}/build-a-python-app-with-cockroachdb.html", + "/${VERSION}/build-a-python-app-with-cockroachdb-sqlalchemy.html", + "/${VERSION}/build-a-python-app-with-cockroachdb-django.html", + "/${VERSION}/build-a-python-app-with-cockroachdb-pony.html" + ] + }, + { + "title": "Ruby", + "urls": [ + "/${VERSION}/build-a-ruby-app-with-cockroachdb.html", + "/${VERSION}/build-a-ruby-app-with-cockroachdb-activerecord.html" + ] + }, + { + "title": "Java", + "urls": [ + "/${VERSION}/build-a-java-app-with-cockroachdb.html", + "/${VERSION}/build-a-java-app-with-cockroachdb-hibernate.html", + "/${VERSION}/build-a-java-app-with-cockroachdb-jooq.html" + ] + }, + { + "title": "Node.js", + "urls": [ + "/${VERSION}/build-a-nodejs-app-with-cockroachdb.html", + "/${VERSION}/build-a-nodejs-app-with-cockroachdb-sequelize.html" + ] + }, + { + "title": "C++", + "urls": [ + "/${VERSION}/build-a-c++-app-with-cockroachdb.html" + ] + }, + { + "title": "C# (.NET)", + "urls": [ + "/${VERSION}/build-a-csharp-app-with-cockroachdb.html" + ] + }, + { + "title": "Clojure", + "urls": [ + "/${VERSION}/build-a-clojure-app-with-cockroachdb.html" + ] + }, + { + "title": "PHP", + "urls": [ + "/${VERSION}/build-a-php-app-with-cockroachdb.html" + ] + }, + { + "title": "Rust", + "urls": [ + "/${VERSION}/build-a-rust-app-with-cockroachdb.html" + ] + } + ] + }, + { + "title": "Multi-Region", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/multi-region-overview.html" + ] + }, + { + "title": "Use-Case", + "urls": [ + "/${VERSION}/multi-region-use-case.html" + ] + }, + { + "title": "Database Schema", + "urls": [ + "/${VERSION}/multi-region-database.html" + ] + }, + { + "title": "Setup", + "urls": [ + "/${VERSION}/multi-region-setup.html" + ] + }, + { + "title": "Application", + "urls": [ + "/${VERSION}/multi-region-application.html" + ] + }, + { + "title": "Deployment", + "urls": [ + "/${VERSION}/multi-region-deployment.html" + ] + } + ] + } + ] + }, + { + "title": "Third-Party Database Tools", + "items": [ + { + "title": "DBeaver GUI", + "urls": [ + "/${VERSION}/dbeaver.html" + ] + }, + { + "title": "Flyway Schema Migrations", + "urls": [ + "/${VERSION}/flyway.html" + ] + }, + { + "title": "IntelliJ IDEA", + "urls": [ + "/${VERSION}/intellij-idea.html" + ] + } + ] + } + ] + }, + { + "title": "Reference", + "is_top_level": true, + "items": [ + { + "title": "Architecture", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/architecture/overview.html" + ] + }, + { + "title": "SQL Layer", + "urls": [ + "/${VERSION}/architecture/sql-layer.html" + ] + }, + { + "title": "Transaction Layer", + "urls": [ + "/${VERSION}/architecture/transaction-layer.html" + ] + }, + { + "title": "Distribution Layer", + "urls": [ + "/${VERSION}/architecture/distribution-layer.html" + ] + }, + { + "title": "Replication Layer", + "urls": [ + "/${VERSION}/architecture/replication-layer.html" + ] + }, + { + "title": "Storage Layer", + "urls": [ + "/${VERSION}/architecture/storage-layer.html" + ] + }, + { + "title": "Life of a Distributed Transaction", + "urls": [ + "/${VERSION}/architecture/life-of-a-distributed-transaction.html" + ] + }, + { + "title": "Reads and Writes Overview", + "urls": [ + "/${VERSION}/architecture/reads-and-writes-overview.html" + ] + } + ] + }, + { + "title": "SQL", + "items": [ + { + "title": "PostgreSQL Compatibility", + "urls": [ + "/${VERSION}/postgresql-compatibility.html" + ] + }, + { + "title": "Client Connection Parameters", + "urls": [ + "/${VERSION}/connection-parameters.html" + ] + }, + { + "title": "SQL Best Practices", + "urls": [ + "/${VERSION}/performance-best-practices-overview.html" + ] + }, + { + "title": "SQL Feature Support", + "items": [ + { + "title": "SQL Support Summary", + "urls": [ + "/${VERSION}/sql-feature-support.html" + ] + }, + { + "title": "Detailed SQL Standard Comparison", + "urls": [ + "/${VERSION}/detailed-sql-support.html" + ] + } + ] + }, + { + "title": "SQL Statements", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/sql-statements.html" + ] + }, + { + "title": "ADD COLUMN", + "urls": [ + "/${VERSION}/add-column.html" + ] + }, + { + "title": "ADD CONSTRAINT", + "urls": [ + "/${VERSION}/add-constraint.html" + ] + }, + { + "title": "ALTER COLUMN", + "urls": [ + "/${VERSION}/alter-column.html" + ] + }, + { + "title": "ALTER DATABASE", + "urls": [ + "/${VERSION}/alter-database.html" + ] + }, + { + "title": "ALTER INDEX", + "urls": [ + "/${VERSION}/alter-index.html" + ] + }, + { + "title": "ALTER PARTITION (Enterprise)", + "urls": [ + "/${VERSION}/alter-partition.html" + ] + }, + { + "title": "ALTER PRIMARY KEY", + "urls": [ + "/${VERSION}/alter-primary-key.html" + ] + }, + { + "title": "ALTER RANGE", + "urls": [ + "/${VERSION}/alter-range.html" + ] + }, + { + "title": "ALTER ROLE", + "urls": [ + "/${VERSION}/alter-role.html" + ] + }, + { + "title": "ALTER SEQUENCE", + "urls": [ + "/${VERSION}/alter-sequence.html" + ] + }, + { + "title": "ALTER TABLE", + "urls": [ + "/${VERSION}/alter-table.html" + ] + }, + { + "title": "ALTER TYPE", + "urls": [ + "/${VERSION}/alter-type.html" + ] + }, + { + "title": "ALTER USER", + "urls": [ + "/${VERSION}/alter-user.html" + ] + }, + { + "title": "EXPERIMENTAL_AUDIT", + "urls": [ + "/${VERSION}/experimental-audit.html" + ] + }, + { + "title": "ALTER VIEW", + "urls": [ + "/${VERSION}/alter-view.html" + ] + }, + { + "title": "BACKUP (Enterprise)", + "urls": [ + "/${VERSION}/backup.html" + ] + }, + { + "title": "BEGIN", + "urls": [ + "/${VERSION}/begin-transaction.html" + ] + }, + { + "title": "CANCEL JOB", + "urls": [ + "/${VERSION}/cancel-job.html" + ] + }, + { + "title": "CANCEL QUERY", + "urls": [ + "/${VERSION}/cancel-query.html" + ] + }, + { + "title": "CANCEL SESSION", + "urls": [ + "/${VERSION}/cancel-session.html" + ] + }, + { + "title": "COMMENT ON", + "urls": [ + "/${VERSION}/comment-on.html" + ] + }, + { + "title": "COMMIT", + "urls": [ + "/${VERSION}/commit-transaction.html" + ] + }, + { + "title": "CONFIGURE ZONE", + "urls": [ + "/${VERSION}/configure-zone.html" + ] + }, + { + "title": "CREATE CHANGEFEED (Enterprise)", + "urls": [ + "/${VERSION}/create-changefeed.html" + ] + }, + { + "title": "CREATE DATABASE", + "urls": [ + "/${VERSION}/create-database.html" + ] + }, + { + "title": "CREATE INDEX", + "urls": [ + "/${VERSION}/create-index.html" + ] + }, + { + "title": "CREATE ROLE", + "urls": [ + "/${VERSION}/create-role.html" + ] + }, + { + "title": "CREATE SEQUENCE", + "urls": [ + "/${VERSION}/create-sequence.html" + ] + }, + { + "title": "CREATE STATISTICS", + "urls": [ + "/${VERSION}/create-statistics.html" + ] + }, + { + "title": "CREATE TABLE", + "urls": [ + "/${VERSION}/create-table.html" + ] + }, + { + "title": "CREATE TABLE AS", + "urls": [ + "/${VERSION}/create-table-as.html" + ] + }, + { + "title": "CREATE USER", + "urls": [ + "/${VERSION}/create-user.html" + ] + }, + { + "title": "CREATE VIEW", + "urls": [ + "/${VERSION}/create-view.html" + ] + }, + { + "title": "DELETE", + "urls": [ + "/${VERSION}/delete.html" + ] + }, + { + "title": "DROP COLUMN", + "urls": [ + "/${VERSION}/drop-column.html" + ] + }, + { + "title": "DROP CONSTRAINT", + "urls": [ + "/${VERSION}/drop-constraint.html" + ] + }, + { + "title": "DROP DATABASE", + "urls": [ + "/${VERSION}/drop-database.html" + ] + }, + { + "title": "DROP INDEX", + "urls": [ + "/${VERSION}/drop-index.html" + ] + }, + { + "title": "DROP ROLE", + "urls": [ + "/${VERSION}/drop-role.html" + ] + }, + { + "title": "DROP SEQUENCE", + "urls": [ + "/${VERSION}/drop-sequence.html" + ] + }, + { + "title": "DROP TABLE", + "urls": [ + "/${VERSION}/drop-table.html" + ] + }, + { + "title": "DROP USER", + "urls": [ + "/${VERSION}/drop-user.html" + ] + }, + { + "title": "DROP VIEW", + "urls": [ + "/${VERSION}/drop-view.html" + ] + }, + { + "title": "EXPERIMENTAL CHANGEFEED FOR", + "urls": [ + "/${VERSION}/changefeed-for.html" + ] + }, + { + "title": "EXPLAIN", + "urls": [ + "/${VERSION}/explain.html" + ] + }, + { + "title": "EXPLAIN ANALYZE", + "urls": [ + "/${VERSION}/explain-analyze.html" + ] + }, + { + "title": "EXPORT (Enterprise)", + "urls": [ + "/${VERSION}/export.html" + ] + }, + { + "title": "GRANT <privileges>", + "urls": [ + "/${VERSION}/grant.html" + ] + }, + { + "title": "GRANT <roles>", + "urls": [ + "/${VERSION}/grant-roles.html" + ] + }, + { + "title": "IMPORT", + "urls": [ + "/${VERSION}/import.html" + ] + }, + { + "title": "IMPORT INTO", + "urls": [ + "/${VERSION}/import-into.html" + ] + }, + { + "title": "INSERT", + "urls": [ + "/${VERSION}/insert.html" + ] + }, + { + "title": "PARTITION BY (Enterprise)", + "urls": [ + "/${VERSION}/partition-by.html" + ] + }, + { + "title": "PAUSE JOB", + "urls": [ + "/${VERSION}/pause-job.html" + ] + }, + { + "title": "RENAME COLUMN", + "urls": [ + "/${VERSION}/rename-column.html" + ] + }, + { + "title": "RENAME CONSTRAINT", + "urls": [ + "/${VERSION}/rename-constraint.html" + ] + }, + { + "title": "RENAME DATABASE", + "urls": [ + "/${VERSION}/rename-database.html" + ] + }, + { + "title": "RENAME INDEX", + "urls": [ + "/${VERSION}/rename-index.html" + ] + }, + { + "title": "RENAME TABLE", + "urls": [ + "/${VERSION}/rename-table.html" + ] + }, + { + "title": "RENAME SEQUENCE", + "urls": [ + "/${VERSION}/rename-sequence.html" + ] + }, + { + "title": "RELEASE SAVEPOINT", + "urls": [ + "/${VERSION}/release-savepoint.html" + ] + }, + { + "title": "RESET <session variable>", + "urls": [ + "/${VERSION}/reset-vars.html" + ] + }, + { + "title": "RESET CLUSTER SETTING", + "urls": [ + "/${VERSION}/reset-cluster-setting.html" + ] + }, + { + "title": "RESTORE (Enterprise)", + "urls": [ + "/${VERSION}/restore.html" + ] + }, + { + "title": "RESUME JOB", + "urls": [ + "/${VERSION}/resume-job.html" + ] + }, + { + "title": "REVOKE <privileges>", + "urls": [ + "/${VERSION}/revoke.html" + ] + }, + { + "title": "REVOKE <roles>", + "urls": [ + "/${VERSION}/revoke-roles.html" + ] + }, + { + "title": "ROLLBACK", + "urls": [ + "/${VERSION}/rollback-transaction.html" + ] + }, + { + "title": "SAVEPOINT", + "urls": [ + "/${VERSION}/savepoint.html" + ] + }, + { + "title": "SELECT", + "urls": [ + "/${VERSION}/select-clause.html" + ] + }, + { + "title": "SELECT FOR UPDATE", + "urls": [ + "/${VERSION}/select-for-update.html" + ] + }, + { + "title": "SET <session variable>", + "urls": [ + "/${VERSION}/set-vars.html" + ] + }, + { + "title": "SET CLUSTER SETTING", + "urls": [ + "/${VERSION}/set-cluster-setting.html" + ] + }, + { + "title": "SET TRANSACTION", + "urls": [ + "/${VERSION}/set-transaction.html" + ] + }, + { + "title": "SHOW <session variables>", + "urls": [ + "/${VERSION}/show-vars.html" + ] + }, + { + "title": "SHOW BACKUP", + "urls": [ + "/${VERSION}/show-backup.html" + ] + }, + { + "title": "SHOW CLUSTER SETTING", + "urls": [ + "/${VERSION}/show-cluster-setting.html" + ] + }, + { + "title": "SHOW COLUMNS", + "urls": [ + "/${VERSION}/show-columns.html" + ] + }, + { + "title": "SHOW CONSTRAINTS", + "urls": [ + "/${VERSION}/show-constraints.html" + ] + }, + { + "title": "SHOW CREATE", + "urls": [ + "/${VERSION}/show-create.html" + ] + }, + { + "title": "SHOW DATABASES", + "urls": [ + "/${VERSION}/show-databases.html" + ] + }, + { + "title": "SHOW GRANTS", + "urls": [ + "/${VERSION}/show-grants.html" + ] + }, + { + "title": "SHOW INDEX", + "urls": [ + "/${VERSION}/show-index.html" + ] + }, + { + "title": "SHOW JOBS", + "urls": [ + "/${VERSION}/show-jobs.html" + ] + }, + { + "title": "SHOW LOCALITY", + "urls": [ + "/${VERSION}/show-locality.html" + ] + }, + { + "title": "SHOW PARTITIONS (Enterprise)", + "urls": [ + "/${VERSION}/show-partitions.html" + ] + }, + { + "title": "SHOW QUERIES", + "urls": [ + "/${VERSION}/show-queries.html" + ] + }, + { + "title": "SHOW RANGES", + "urls": [ + "/${VERSION}/show-ranges.html" + ] + }, + { + "title": "SHOW RANGE FOR ROW", + "urls": [ + "/${VERSION}/show-range-for-row.html" + ] + }, + { + "title": "SHOW ROLES", + "urls": [ + "/${VERSION}/show-roles.html" + ] + }, + { + "title": "SHOW SCHEMAS", + "urls": [ + "/${VERSION}/show-schemas.html" + ] + }, + { + "title": "SHOW SEQUENCES", + "urls": [ + "/${VERSION}/show-sequences.html" + ] + }, + { + "title": "SHOW SESSIONS", + "urls": [ + "/${VERSION}/show-sessions.html" + ] + }, + { + "title": "SHOW STATISTICS", + "urls": [ + "/${VERSION}/show-statistics.html" + ] + }, + { + "title": "SHOW SAVEPOINT STATUS", + "urls": [ + "/${VERSION}/show-savepoint-status.html" + ] + }, + { + "title": "SHOW TABLES", + "urls": [ + "/${VERSION}/show-tables.html" + ] + }, + { + "title": "SHOW TRACE FOR SESSION", + "urls": [ + "/${VERSION}/show-trace.html" + ] + }, + { + "title": "SHOW USERS", + "urls": [ + "/${VERSION}/show-users.html" + ] + }, + { + "title": "SHOW ZONE CONFIGURATIONS", + "urls": [ + "/${VERSION}/show-zone-configurations.html" + ] + }, + { + "title": "SPLIT AT", + "urls": [ + "/${VERSION}/split-at.html" + ] + }, + { + "title": "TRUNCATE", + "urls": [ + "/${VERSION}/truncate.html" + ] + }, + { + "title": "UNSPLIT AT", + "urls": [ + "/${VERSION}/unsplit-at.html" + ] + }, + { + "title": "UPDATE", + "urls": [ + "/${VERSION}/update.html" + ] + }, + { + "title": "UPSERT", + "urls": [ + "/${VERSION}/upsert.html" + ] + }, + { + "title": "VALIDATE CONSTRAINT", + "urls": [ + "/${VERSION}/validate-constraint.html" + ] + } + ] + }, + { + "title": "Data Types", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/data-types.html" + ] + }, + { + "title": "ARRAY", + "urls": [ + "/${VERSION}/array.html" + ] + }, + { + "title": "BIT", + "urls": [ + "/${VERSION}/bit.html" + ] + }, + { + "title": "BOOL", + "urls": [ + "/${VERSION}/bool.html" + ] + }, + { + "title": "BYTES", + "urls": [ + "/${VERSION}/bytes.html" + ] + }, + { + "title": "COLLATE", + "urls": [ + "/${VERSION}/collate.html" + ] + }, + { + "title": "DATE", + "urls": [ + "/${VERSION}/date.html" + ] + }, + { + "title": "DECIMAL", + "urls": [ + "/${VERSION}/decimal.html" + ] + }, + { + "title": "FLOAT", + "urls": [ + "/${VERSION}/float.html" + ] + }, + { + "title": "INET", + "urls": [ + "/${VERSION}/inet.html" + ] + }, + { + "title": "INT", + "urls": [ + "/${VERSION}/int.html" + ] + }, + { + "title": "INTERVAL", + "urls": [ + "/${VERSION}/interval.html" + ] + }, + { + "title": "JSONB", + "urls": [ + "/${VERSION}/jsonb.html" + ] + }, + { + "title": "SERIAL", + "urls": [ + "/${VERSION}/serial.html" + ] + }, + { + "title": "STRING", + "urls": [ + "/${VERSION}/string.html" + ] + }, + { + "title": "TIME", + "urls": [ + "/${VERSION}/time.html" + ] + }, + { + "title": "TIMESTAMP", + "urls": [ + "/${VERSION}/timestamp.html" + ] + }, + { + "title": "UUID", + "urls": [ + "/${VERSION}/uuid.html" + ] + } + ] + }, + { + "title": "Constraints", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/constraints.html" + ] + }, + { + "title": "Check", + "urls": [ + "/${VERSION}/check.html" + ] + }, + { + "title": "Default Value", + "urls": [ + "/${VERSION}/default-value.html" + ] + }, + { + "title": "Foreign Key", + "urls": [ + "/${VERSION}/foreign-key.html" + ] + }, + { + "title": "Not Null", + "urls": [ + "/${VERSION}/not-null.html" + ] + }, + { + "title": "Primary Key", + "urls": [ + "/${VERSION}/primary-key.html" + ] + }, + { + "title": "Unique", + "urls": [ + "/${VERSION}/unique.html" + ] + } + ] + }, + { + "title": "Functions and Operators", + "urls": [ + "/${VERSION}/functions-and-operators.html" + ] + }, + { + "title": "SQL Syntax", + "items": [ + { + "title": "Full SQL Grammar", + "urls": [ + "/${VERSION}/sql-grammar.html" + ] + }, + { + "title": "Keywords & Identifiers", + "urls": [ + "/${VERSION}/keywords-and-identifiers.html" + ] + }, + { + "title": "Constants", + "urls": [ + "/${VERSION}/sql-constants.html" + ] + }, + { + "title": "Selection Queries", + "urls": [ + "/${VERSION}/selection-queries.html" + ] + }, + { + "title": "Ordering Query Results", + "urls": [ + "/${VERSION}/query-order.html" + ] + }, + { + "title": "Limiting Query Results", + "urls": [ + "/${VERSION}/limit-offset.html" + ] + }, + { + "title": "Table Expressions", + "urls": [ + "/${VERSION}/table-expressions.html" + ] + }, + { + "title": "Common Table Expressions", + "urls": [ + "/${VERSION}/common-table-expressions.html" + ] + }, + { + "title": "Join Expressions", + "urls": [ + "/${VERSION}/joins.html" + ] + }, + { + "title": "Computed Columns", + "urls": [ + "/${VERSION}/computed-columns.html" + ] + }, + { + "title": "Scalar Expressions", + "urls": [ + "/${VERSION}/scalar-expressions.html" + ] + }, + { + "title": "Subqueries", + "urls": [ + "/${VERSION}/subqueries.html" + ] + }, + { + "title": "Temporary Tables", + "urls": [ + "/${VERSION}/temporary-tables.html" + ] + }, + { + "title": "Name Resolution", + "urls": [ + "/${VERSION}/sql-name-resolution.html" + ] + }, + { + "title": "AS OF SYSTEM TIME", + "urls": [ + "/${VERSION}/as-of-system-time.html" + ] + }, + { + "title": "NULL Handling", + "urls": [ + "/${VERSION}/null-handling.html" + ] + } + ] + }, + { + "title": "Transactions", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/transactions.html" + ] + }, + { + "title": "Advanced Client-side Transaction Retries", + "urls": [ + "/${VERSION}/advanced-client-side-transaction-retries.html" + ] + } + ] + }, + { + "title": "Views", + "urls": [ + "/${VERSION}/views.html" + ] + }, + { + "title": "Window Functions", + "urls": [ + "/${VERSION}/window-functions.html" + ] + }, + { + "title": "Performance Optimization", + "items": [ + { + "title": "SQL Best Practices", + "urls": [ + "/${VERSION}/performance-best-practices-overview.html" + ] + }, + { + "title": "Indexes", + "urls": [ + "/${VERSION}/indexes.html" + ] + }, + { + "title": "Inverted Indexes", + "urls": [ + "/${VERSION}/inverted-indexes.html" + ] + }, + { + "title": "Column Families", + "urls": [ + "/${VERSION}/column-families.html" + ] + }, + { + "title": "Interleaved Tables", + "urls": [ + "/${VERSION}/interleave-in-parent.html" + ] + }, + { + "title": "Cost-Based Optimizer", + "urls": [ + "/${VERSION}/cost-based-optimizer.html" + ] + }, + { + "title": "Vectorized Execution", + "urls": [ + "/${VERSION}/vectorized-execution.html" + ] + } + ] + }, + { + "title": "Information Schema", + "urls": [ + "/${VERSION}/information-schema.html" + ] + }, + { + "title": "Experimental Features", + "urls": [ + "/${VERSION}/experimental-features.html" + ] + } + ] + }, + { + "title": "CLI", + "items": [ + { + "title": "Overview", + "urls": [ + "/${VERSION}/cockroach-commands.html" + ] + }, + { + "title": "cockroach start", + "urls": [ + "/${VERSION}/cockroach-start.html" + ] + }, + { + "title": "cockroach init", + "urls": [ + "/${VERSION}/cockroach-init.html" + ] + }, + { + "title": "cockroach start-single-node", + "urls": [ + "/${VERSION}/cockroach-start-single-node.html" + ] + }, + { + "title": "cockroach cert", + "urls": [ + "/${VERSION}/cockroach-cert.html" + ] + }, + { + "title": "cockroach quit", + "urls": [ + "/${VERSION}/cockroach-quit.html" + ] + }, + { + "title": "cockroach sql", + "urls": [ + "/${VERSION}/cockroach-sql.html" + ] + }, + { + "title": "cockroach sqlfmt", + "urls": [ + "/${VERSION}/cockroach-sqlfmt.html" + ] + }, + { + "title": "cockroach node", + "urls": [ + "/${VERSION}/cockroach-node.html" + ] + }, + { + "title": "cockroach nodelocal upload", + "urls": [ + "/${VERSION}/cockroach-nodelocal-upload.html" + ] + }, + { + "title": "cockroach dump", + "urls": [ + "/${VERSION}/cockroach-dump.html" + ] + }, + { + "title": "cockroach demo", + "urls": [ + "/${VERSION}/cockroach-demo.html" + ] + }, + { + "title": "cockroach debug ballast", + "urls": [ + "/${VERSION}/cockroach-debug-ballast.html" + ] + }, + { + "title": "cockroach debug encryption-active-key", + "urls": [ + "/${VERSION}/cockroach-debug-encryption-active-key.html" + ] + }, + { + "title": "cockroach debug merge-logs", + "urls": [ + "/${VERSION}/cockroach-debug-merge-logs.html" + ] + }, + { + "title": "cockroach debug zip", + "urls": [ + "/${VERSION}/cockroach-debug-zip.html" + ] + }, + { + "title": "cockroach gen", + "urls": [ + "/${VERSION}/cockroach-gen.html" + ] + }, + { + "title": "cockroach version", + "urls": [ + "/${VERSION}/cockroach-version.html" + ] + }, + { + "title": "cockroach workload", + "urls": [ + "/${VERSION}/cockroach-workload.html" + ] + } + ] + }, + { + "title": "Cluster Settings", + "items": [ + { + "title": "Cluster Settings Overview", + "urls": [ + "/${VERSION}/cluster-settings.html" + ] + }, + { + "title": "Cost-Based Optimizer", + "urls": [ + "/${VERSION}/cost-based-optimizer.html" + ] + }, + { + "title": "Follower Reads", + "urls": [ + "/${VERSION}/follower-reads.html" + ] + }, + { + "title": "Load-Based Splitting", + "urls": [ + "/${VERSION}/load-based-splitting.html" + ] + }, + { + "title": "Range Merges", + "urls": [ + "/${VERSION}/range-merges.html" + ] + } + ] + }, + { + "title": "Admin UI", + "items": [ + { + "title": "Admin UI Overview", + "urls": [ + "/${VERSION}/admin-ui-overview.html" + ] + }, + { + "title": "Cluster Overview Page", + "urls": [ + "/${VERSION}/admin-ui-cluster-overview-page.html" + ] + }, + { + "title": "Overview Dashboard", + "urls": [ + "/${VERSION}/admin-ui-overview-dashboard.html" + ] + }, + { + "title": "Hardware Dashboard", + "urls": [ + "/${VERSION}/admin-ui-hardware-dashboard.html" + ] + }, + { + "title": "Runtime Dashboard", + "urls": [ + "/${VERSION}/admin-ui-runtime-dashboard.html" + ] + }, + { + "title": "SQL Dashboard", + "urls": [ + "/${VERSION}/admin-ui-sql-dashboard.html" + ] + }, + { + "title": "Storage Dashboard", + "urls": [ + "/${VERSION}/admin-ui-storage-dashboard.html" + ] + }, + { + "title": "Replication Dashboard", + "urls": [ + "/${VERSION}/admin-ui-replication-dashboard.html" + ] + }, + { + "title": "Changefeeds Dashboard", + "urls": [ + "/${VERSION}/admin-ui-cdc-dashboard.html" + ] + }, + { + "title": "Databases Page", + "urls": [ + "/${VERSION}/admin-ui-databases-page.html" + ] + }, + { + "title": "Statements Page", + "urls": [ + "/${VERSION}/admin-ui-statements-page.html" + ] + }, + { + "title": "Network Latency Page", + "urls": [ + "/${VERSION}/admin-ui-network-latency-page.html" + ] + }, + { + "title": "Jobs Page", + "urls": [ + "/${VERSION}/admin-ui-jobs-page.html" + ] + }, + { + "title": "Advanced Debugging Page", + "urls": [ + "/${VERSION}/admin-ui-debug-pages.html" + ] + }, + { + "title": "Custom Chart Debug Page", + "urls": [ + "/${VERSION}/admin-ui-custom-chart-debug-page.html" + ] + } + ] + }, + { + "title": "Third-Party Database Tools", + "urls": [ + "/${VERSION}/third-party-database-tools.html" + ] + }, + { + "title": "Diagnostics Reporting", + "urls": [ + "/${VERSION}/diagnostics-reporting.html" + ] + }, + { + "title": "Sample Applications", + "items": [ + { + "title": "MovR: Vehicle-Sharing App", + "urls": [ + "/${VERSION}/movr.html" + ] + } + ] + } + ] + }, + { + "title": "FAQs", + "is_top_level": true, + "items": [ + { + "title": "Product FAQs", + "urls": [ + "/${VERSION}/frequently-asked-questions.html" + ] + }, + { + "title": "SQL FAQs", + "urls": [ + "/${VERSION}/sql-faqs.html" + ] + }, + { + "title": "Operational FAQs", + "urls": [ + "/${VERSION}/operational-faqs.html" + ] + }, + { + "title": "Availability FAQs", + "urls": [ + "/${VERSION}/multi-active-availability.html" + ] + }, + { + "title": "Licensing FAQs", + "urls": [ + "/${VERSION}/licensing-faqs.html" + ] + }, + { + "title": "CockroachDB in Comparison", + "urls": [ + "/${VERSION}/cockroachdb-in-comparison.html" + ] + } + ] + }, + {% include sidebar-releases.json %} +] diff --git a/_includes/v20.2/admin-ui-custom-chart-debug-page-00.html b/_includes/v20.2/admin-ui-custom-chart-debug-page-00.html new file mode 100644 index 00000000000..36e0764df99 --- /dev/null +++ b/_includes/v20.2/admin-ui-custom-chart-debug-page-00.html @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Column + + Description +
+ Metric Name + + How the system refers to this metric, e.g., sql.bytesin. +
+ Downsampler + +

+ The "Downsampler" operation is used to combine the individual datapoints over the longer period into a single datapoint. We store one data point every ten seconds, but for queries over long time spans the backend lowers the resolution of the returned data, perhaps only returning one data point for every minute, five minutes, or even an entire hour in the case of the 30 day view. +

+

+ Options: +

    +
  • AVG: Returns the average value over the time period.
  • +
  • MIN: Returns the lowest value seen.
  • +
  • MAX: Returns the highest value seen.
  • +
  • SUM: Returns the sum of all values seen.
  • +
+

+
+ Aggregator + +

+ Used to combine data points from different nodes. It has the same operations available as the Downsampler. +

+

+ Options: +

    +
  • AVG: Returns the average value over the time period.
  • +
  • MIN: Returns the lowest value seen.
  • +
  • MAX: Returns the highest value seen.
  • +
  • SUM: Returns the sum of all values seen.
  • +
+

+
+ Rate + +

+ Determines how to display the rate of change during the selected time period. +

+

+ Options: + +

    +
  • + Normal: Returns the actual recorded value. +
  • +
  • + Rate: Returns the rate of change of the value per second. +
  • +
  • + Non-negative Rate: Returns the rate-of-change, but returns 0 instead of negative values. A large number of the stats we track are actually tracked as monotonically increasing counters so each sample is just the total value of that counter. The rate of change of that counter represents the rate of events being counted, which is usually what you want to graph. "Non-negative Rate" is needed because the counters are stored in memory, and thus if a node resets it goes back to zero (whereas normally they only increase). +
  • +
+

+
+ Source + + The set of nodes being queried, which is either: +
    +
  • + The entire cluster. +
  • +
  • + A single, named node. +
  • +
+
+ Per Node + + If checked, the chart will show a line for each node's value of this metric. +
diff --git a/_includes/v20.2/admin-ui/admin-ui-log-files.md b/_includes/v20.2/admin-ui/admin-ui-log-files.md new file mode 100644 index 00000000000..51ed9c3aee5 --- /dev/null +++ b/_includes/v20.2/admin-ui/admin-ui-log-files.md @@ -0,0 +1,7 @@ +Log files can be accessed using the Admin UI, which displays them in JSON format. + +1. [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and then click [**Advanced Debug**](admin-ui-debug-pages.html) in the left-hand navigation. + +2. Under **Raw Status Endpoints (JSON)**, click **Log Files** to view the JSON of all collected logs. + +3. Copy one of the log filenames. Then click **Specific Log File** and replace the `cockroach.log` placeholder in the URL with the filename. \ No newline at end of file diff --git a/_includes/v20.2/admin-ui/admin-ui-metrics-navigation.md b/_includes/v20.2/admin-ui/admin-ui-metrics-navigation.md new file mode 100644 index 00000000000..6516c21dfce --- /dev/null +++ b/_includes/v20.2/admin-ui/admin-ui-metrics-navigation.md @@ -0,0 +1,5 @@ +## Dashboard navigation + +Use the **Graph** menu to display metrics for your entire cluster or for a specific node. + +To the right of the Graph and Dashboard menus, a range selector allows you to filter the view for a predefined timeframe or custom date/time range. Use the navigation buttons to move to the previous, next, or current timeframe. Note that the active timeframe is reflected in the URL and can be easily shared. \ No newline at end of file diff --git a/_includes/v20.2/admin-ui/logical-bytes.md b/_includes/v20.2/admin-ui/logical-bytes.md new file mode 100644 index 00000000000..e85f04cea92 --- /dev/null +++ b/_includes/v20.2/admin-ui/logical-bytes.md @@ -0,0 +1 @@ +Logical bytes reflect the approximate number of bytes stored in the database. This value may deviate from the number of physical bytes on disk, due to factors such as compression and [write amplification](https://en.wikipedia.org/wiki/Write_amplification). \ No newline at end of file diff --git a/_includes/v20.2/app/BasicExample.java b/_includes/v20.2/app/BasicExample.java new file mode 100644 index 00000000000..d63c30f6aba --- /dev/null +++ b/_includes/v20.2/app/BasicExample.java @@ -0,0 +1,437 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setSsl(true); + ds.setSslMode("require"); + ds.setSslCert("certs/client.maxroach.crt"); + ds.setSslKey("certs/client.maxroach.key.pk8"); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in these accounts? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/_includes/v20.2/app/BasicSample.java b/_includes/v20.2/app/BasicSample.java new file mode 100644 index 00000000000..25d326dd4e0 --- /dev/null +++ b/_includes/v20.2/app/BasicSample.java @@ -0,0 +1,55 @@ +import java.sql.*; +import java.util.Properties; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicSample.java && java BasicSample +*/ + +public class BasicSample { + public static void main(String[] args) + throws ClassNotFoundException, SQLException { + + // Load the Postgres JDBC driver. + Class.forName("org.postgresql.Driver"); + + // Connect to the "bank" database. + Properties props = new Properties(); + props.setProperty("user", "maxroach"); + props.setProperty("sslmode", "require"); + props.setProperty("sslrootcert", "certs/ca.crt"); + props.setProperty("sslkey", "certs/client.maxroach.key.pk8"); + props.setProperty("sslcert", "certs/client.maxroach.crt"); + props.setProperty("ApplicationName", "roachtest"); + + Connection db = DriverManager + .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); + + try { + // Create the "accounts" table. + db.createStatement() + .execute("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); + + // Insert two rows into the "accounts" table. + db.createStatement() + .execute("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); + + // Print out the balances. + System.out.println("Initial balances:"); + ResultSet res = db.createStatement() + .executeQuery("SELECT id, balance FROM accounts"); + while (res.next()) { + System.out.printf("\taccount %s: %s\n", + res.getInt("id"), + res.getInt("balance")); + } + } finally { + // Close the database connection. + db.close(); + } + } +} diff --git a/_includes/v20.2/app/TxnSample.java b/_includes/v20.2/app/TxnSample.java new file mode 100644 index 00000000000..624e67c80d5 --- /dev/null +++ b/_includes/v20.2/app/TxnSample.java @@ -0,0 +1,148 @@ +import java.sql.*; +import java.util.Properties; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac TxnSample.java && java TxnSample +*/ + +// Ambiguous whether the transaction committed or not. +class AmbiguousCommitException extends SQLException{ + public AmbiguousCommitException(Throwable cause) { + super(cause); + } +} + +class InsufficientBalanceException extends Exception {} + +class AccountNotFoundException extends Exception { + public int account; + public AccountNotFoundException(int account) { + this.account = account; + } +} + +// A simple interface that provides a retryable lambda expression. +interface RetryableTransaction { + public void run(Connection conn) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException; +} + +public class TxnSample { + public static RetryableTransaction transferFunds(int from, int to, int amount) { + return new RetryableTransaction() { + public void run(Connection conn) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException { + + // Check the current balance. + ResultSet res = conn.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + from); + if(!res.next()) { + throw new AccountNotFoundException(from); + } + + int balance = res.getInt("balance"); + if(balance < from) { + throw new InsufficientBalanceException(); + } + + // Perform the transfer. + conn.createStatement() + .executeUpdate("UPDATE accounts SET balance = balance - " + + amount + " where id = " + from); + conn.createStatement() + .executeUpdate("UPDATE accounts SET balance = balance + " + + amount + " where id = " + to); + } + }; + } + + public static void retryTransaction(Connection conn, RetryableTransaction tx) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException { + + Savepoint sp = conn.setSavepoint("cockroach_restart"); + while(true) { + boolean releaseAttempted = false; + try { + tx.run(conn); + releaseAttempted = true; + conn.releaseSavepoint(sp); + break; + } + catch(SQLException e) { + String sqlState = e.getSQLState(); + + // Check if the error code indicates a SERIALIZATION_FAILURE. + if(sqlState.equals("40001")) { + // Signal the database that we will attempt a retry. + conn.rollback(sp); + } else if(releaseAttempted) { + throw new AmbiguousCommitException(e); + } else { + throw e; + } + } + } + conn.commit(); + } + + public static void main(String[] args) + throws ClassNotFoundException, SQLException { + + // Load the Postgres JDBC driver. + Class.forName("org.postgresql.Driver"); + + // Connect to the 'bank' database. + Properties props = new Properties(); + props.setProperty("user", "maxroach"); + props.setProperty("sslmode", "require"); + props.setProperty("sslrootcert", "certs/ca.crt"); + props.setProperty("sslkey", "certs/client.maxroach.key.pk8"); + props.setProperty("sslcert", "certs/client.maxroach.crt"); + props.setProperty("ApplicationName", "roachtest"); + + Connection db = DriverManager + .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); + + + try { + // We need to turn off autocommit mode to allow for + // multi-statement transactions. + db.setAutoCommit(false); + + // Perform the transfer. This assumes the 'accounts' + // table has already been created in the database. + RetryableTransaction transfer = transferFunds(1, 2, 100); + retryTransaction(db, transfer); + + // Check balances after transfer. + db.setAutoCommit(true); + ResultSet res = db.createStatement() + .executeQuery("SELECT id, balance FROM accounts"); + while (res.next()) { + System.out.printf("\taccount %s: %s\n", res.getInt("id"), + res.getInt("balance")); + } + + } catch(InsufficientBalanceException e) { + System.out.println("Insufficient balance"); + } catch(AccountNotFoundException e) { + System.out.println("No users in the table with id " + e.account); + } catch(AmbiguousCommitException e) { + System.out.println("Ambiguous result encountered: " + e); + } catch(SQLException e) { + System.out.println("SQLException encountered:" + e); + } finally { + // Close the database connection. + db.close(); + } + } +} diff --git a/_includes/v20.2/app/activerecord-basic-sample.rb b/_includes/v20.2/app/activerecord-basic-sample.rb new file mode 100644 index 00000000000..f1d35e1de3a --- /dev/null +++ b/_includes/v20.2/app/activerecord-basic-sample.rb @@ -0,0 +1,48 @@ +require 'active_record' +require 'activerecord-cockroachdb-adapter' +require 'pg' + +# Connect to CockroachDB through ActiveRecord. +# In Rails, this configuration would go in config/database.yml as usual. +ActiveRecord::Base.establish_connection( + adapter: 'cockroachdb', + username: 'maxroach', + database: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'require', + sslrootcert: 'certs/ca.crt', + sslkey: 'certs/client.maxroach.key', + sslcert: 'certs/client.maxroach.crt' +) + + +# Define the Account model. +# In Rails, this would go in app/models/ as usual. +class Account < ActiveRecord::Base + validates :id, presence: true + validates :balance, presence: true +end + +# Define a migration for the accounts table. +# In Rails, this would go in db/migrate/ as usual. +class Schema < ActiveRecord::Migration[5.0] + def change + create_table :accounts, force: true do |t| + t.integer :balance + end + end +end + +# Run the schema migration by hand. +# In Rails, this would be done via rake db:migrate as usual. +Schema.new.change() + +# Create two accounts, inserting two rows into the accounts table. +Account.create(id: 1, balance: 1000) +Account.create(id: 2, balance: 250) + +# Retrieve accounts and print out the balances +Account.all.each do |acct| + puts "#{acct.id} #{acct.balance}" +end diff --git a/_includes/v20.2/app/basic-sample.c b/_includes/v20.2/app/basic-sample.c new file mode 100644 index 00000000000..e69de29bb2d diff --git a/_includes/v20.2/app/basic-sample.clj b/_includes/v20.2/app/basic-sample.clj new file mode 100644 index 00000000000..10c98fff2ba --- /dev/null +++ b/_includes/v20.2/app/basic-sample.clj @@ -0,0 +1,35 @@ +(ns test.test + (:require [clojure.java.jdbc :as j] + [test.util :as util])) + +;; Define the connection parameters to the cluster. +(def db-spec {:dbtype "postgresql" + :dbname "bank" + :host "localhost" + :port "26257" + :ssl true + :sslmode "require" + :sslcert "certs/client.maxroach.crt" + :sslkey "certs/client.maxroach.key.pk8" + :user "maxroach"}) + +(defn test-basic [] + ;; Connect to the cluster and run the code below with + ;; the connection object bound to 'conn'. + (j/with-db-connection [conn db-spec] + + ;; Insert two rows into the "accounts" table. + (j/insert! conn :accounts {:id 1 :balance 1000}) + (j/insert! conn :accounts {:id 2 :balance 250}) + + ;; Print out the balances. + (println "Initial balances:") + (->> (j/query conn ["SELECT id, balance FROM accounts"]) + (map println) + doall) + + )) + + +(defn -main [& args] + (test-basic)) diff --git a/_includes/v20.2/app/basic-sample.cpp b/_includes/v20.2/app/basic-sample.cpp new file mode 100644 index 00000000000..67b6c1d1062 --- /dev/null +++ b/_includes/v20.2/app/basic-sample.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; + +int main() { + try { + // Connect to the "bank" database. + pqxx::connection c("dbname=bank user=maxroach sslmode=require sslkey=certs/client.maxroach.key sslcert=certs/client.maxroach.crt port=26257 host=localhost"); + + pqxx::nontransaction w(c); + + // Create the "accounts" table. + w.exec("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); + + // Insert two rows into the "accounts" table. + w.exec("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); + + // Print out the balances. + cout << "Initial balances:" << endl; + pqxx::result r = w.exec("SELECT id, balance FROM accounts"); + for (auto row : r) { + cout << row[0].as() << ' ' << row[1].as() << endl; + } + + w.commit(); // Note this doesn't doesn't do anything + // for a nontransaction, but is still required. + } + catch (const exception &e) { + cerr << e.what() << endl; + return 1; + } + cout << "Success" << endl; + return 0; +} diff --git a/_includes/v20.2/app/basic-sample.cs b/_includes/v20.2/app/basic-sample.cs new file mode 100644 index 00000000000..d23bcf9eb11 --- /dev/null +++ b/_includes/v20.2/app/basic-sample.cs @@ -0,0 +1,101 @@ +using System; +using System.Data; +using System.Security.Cryptography.X509Certificates; +using System.Net.Security; +using Npgsql; + +namespace Cockroach +{ + class MainClass + { + static void Main(string[] args) + { + var connStringBuilder = new NpgsqlConnectionStringBuilder(); + connStringBuilder.Host = "localhost"; + connStringBuilder.Port = 26257; + connStringBuilder.SslMode = SslMode.Require; + connStringBuilder.Username = "maxroach"; + connStringBuilder.Database = "bank"; + Simple(connStringBuilder.ConnectionString); + } + + static void Simple(string connString) + { + using (var conn = new NpgsqlConnection(connString)) + { + conn.ProvideClientCertificatesCallback += ProvideClientCertificatesCallback; + conn.UserCertificateValidationCallback += UserCertificateValidationCallback; + conn.Open(); + + // Create the "accounts" table. + new NpgsqlCommand("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", conn).ExecuteNonQuery(); + + // Insert two rows into the "accounts" table. + using (var cmd = new NpgsqlCommand()) + { + cmd.Connection = conn; + cmd.CommandText = "UPSERT INTO accounts(id, balance) VALUES(@id1, @val1), (@id2, @val2)"; + cmd.Parameters.AddWithValue("id1", 1); + cmd.Parameters.AddWithValue("val1", 1000); + cmd.Parameters.AddWithValue("id2", 2); + cmd.Parameters.AddWithValue("val2", 250); + cmd.ExecuteNonQuery(); + } + + // Print out the balances. + System.Console.WriteLine("Initial balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + } + } + + static void ProvideClientCertificatesCallback(X509CertificateCollection clientCerts) + { + // To be able to add a certificate with a private key included, we must convert it to + // a PKCS #12 format. The following openssl command does this: + // openssl pkcs12 -password pass: -inkey client.maxroach.key -in client.maxroach.crt -export -out client.maxroach.pfx + // As of 2018-12-10, you need to provide a password for this to work on macOS. + // See https://github.com/dotnet/corefx/issues/24225 + + // Note that the password used during X509 cert creation below + // must match the password used in the openssl command above. + clientCerts.Add(new X509Certificate2("client.maxroach.pfx", "pass")); + } + + // By default, .Net does all of its certificate verification using the system certificate store. + // This callback is necessary to validate the server certificate against a CA certificate file. + static bool UserCertificateValidationCallback(object sender, X509Certificate certificate, X509Chain defaultChain, SslPolicyErrors defaultErrors) + { + X509Certificate2 caCert = new X509Certificate2("ca.crt"); + X509Chain caCertChain = new X509Chain(); + caCertChain.ChainPolicy = new X509ChainPolicy() + { + RevocationMode = X509RevocationMode.NoCheck, + RevocationFlag = X509RevocationFlag.EntireChain + }; + caCertChain.ChainPolicy.ExtraStore.Add(caCert); + + X509Certificate2 serverCert = new X509Certificate2(certificate); + + caCertChain.Build(serverCert); + if (caCertChain.ChainStatus.Length == 0) + { + // No errors + return true; + } + + foreach (X509ChainStatus status in caCertChain.ChainStatus) + { + // Check if we got any errors other than UntrustedRoot (which we will always get if we don't install the CA cert to the system store) + if (status.Status != X509ChainStatusFlags.UntrustedRoot) + { + return false; + } + } + return true; + } + + } +} diff --git a/_includes/v20.2/app/basic-sample.go b/_includes/v20.2/app/basic-sample.go new file mode 100644 index 00000000000..6e22c858dbb --- /dev/null +++ b/_includes/v20.2/app/basic-sample.go @@ -0,0 +1,46 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/lib/pq" +) + +func main() { + // Connect to the "bank" database. + db, err := sql.Open("postgres", + "postgresql://maxroach@localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key&sslcert=certs/client.maxroach.crt") + if err != nil { + log.Fatal("error connecting to the database: ", err) + } + defer db.Close() + + // Create the "accounts" table. + if _, err := db.Exec( + "CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); err != nil { + log.Fatal(err) + } + + // Insert two rows into the "accounts" table. + if _, err := db.Exec( + "INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); err != nil { + log.Fatal(err) + } + + // Print out the balances. + rows, err := db.Query("SELECT id, balance FROM accounts") + if err != nil { + log.Fatal(err) + } + defer rows.Close() + fmt.Println("Initial balances:") + for rows.Next() { + var id, balance int + if err := rows.Scan(&id, &balance); err != nil { + log.Fatal(err) + } + fmt.Printf("%d %d\n", id, balance) + } +} diff --git a/_includes/v20.2/app/basic-sample.js b/_includes/v20.2/app/basic-sample.js new file mode 100644 index 00000000000..4e86cb2cbca --- /dev/null +++ b/_includes/v20.2/app/basic-sample.js @@ -0,0 +1,63 @@ +var async = require('async'); +var fs = require('fs'); +var pg = require('pg'); + +// Connect to the "bank" database. +var config = { + user: 'maxroach', + host: 'localhost', + database: 'bank', + port: 26257, + ssl: { + ca: fs.readFileSync('certs/ca.crt') + .toString(), + key: fs.readFileSync('certs/client.maxroach.key') + .toString(), + cert: fs.readFileSync('certs/client.maxroach.crt') + .toString() + } +}; + +// Create a pool. +var pool = new pg.Pool(config); + +pool.connect(function (err, client, done) { + + // Close communication with the database and exit. + var finish = function () { + done(); + process.exit(); + }; + + if (err) { + console.error('could not connect to cockroachdb', err); + finish(); + } + async.waterfall([ + function (next) { + // Create the 'accounts' table. + client.query('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT);', next); + }, + function (results, next) { + // Insert two rows into the 'accounts' table. + client.query('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250);', next); + }, + function (results, next) { + // Print out account balances. + client.query('SELECT id, balance FROM accounts;', next); + }, + ], + function (err, results) { + if (err) { + console.error('Error inserting into and selecting from accounts: ', err); + finish(); + } + + console.log('Initial balances:'); + results.rows.forEach(function (row) { + console.log(row); + }); + + finish(); + }); +}); diff --git a/_includes/v20.2/app/basic-sample.php b/_includes/v20.2/app/basic-sample.php new file mode 100644 index 00000000000..4edae09b12a --- /dev/null +++ b/_includes/v20.2/app/basic-sample.php @@ -0,0 +1,20 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_PERSISTENT => true + )); + + $dbh->exec('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)'); + + print "Account balances:\r\n"; + foreach ($dbh->query('SELECT id, balance FROM accounts') as $row) { + print $row['id'] . ': ' . $row['balance'] . "\r\n"; + } +} catch (Exception $e) { + print $e->getMessage() . "\r\n"; + exit(1); +} +?> diff --git a/_includes/v20.2/app/basic-sample.py b/_includes/v20.2/app/basic-sample.py new file mode 100644 index 00000000000..189d8c91797 --- /dev/null +++ b/_includes/v20.2/app/basic-sample.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 + +import psycopg2 +import psycopg2.errorcodes +import time +import logging +import random + + +def create_accounts(conn): + with conn.cursor() as cur: + cur.execute('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)') + cur.execute('UPSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)') + logging.debug("create_accounts(): status message: {}".format(cur.statusmessage)) + conn.commit() + + +def print_balances(conn): + with conn.cursor() as cur: + cur.execute("SELECT id, balance FROM accounts") + logging.debug("print_balances(): status message: {}".format(cur.statusmessage)) + rows = cur.fetchall() + conn.commit() + print("Balances at {}".format(time.asctime())) + for row in rows: + print([str(cell) for cell in row]) + + +def delete_accounts(conn): + with conn.cursor() as cur: + cur.execute("DELETE FROM bank.accounts") + logging.debug("delete_accounts(): status message: {}".format(cur.statusmessage)) + conn.commit() + + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn, op): + retries = 0 + max_retries = 3 + with conn: + while True: + retries +=1 + if retries == max_retries: + err_msg = "Transaction did not succeed after {} retries".format(max_retries) + raise ValueError(err_msg) + + try: + op(conn) + + # If we reach this point, we were able to commit, so we break + # from the retry loop. + break + except psycopg2.Error as e: + logging.debug("e.pgcode: {}".format(e.pgcode)) + if e.pgcode == '40001': + # This is a retry error, so we roll back the current + # transaction and sleep for a bit before retrying. The + # sleep time increases for each failed transaction. + conn.rollback() + logging.debug("EXECUTE SERIALIZATION_FAILURE BRANCH") + sleep_ms = (2**retries) * 0.1 * (random.random() + 0.5) + logging.debug("Sleeping {} seconds".format(sleep_ms)) + time.sleep(sleep_ms) + continue + else: + logging.debug("EXECUTE NON-SERIALIZATION_FAILURE BRANCH") + raise e + + +# This function is used to test the transaction retry logic. It can be deleted +# from production code. +def test_retry_loop(conn): + with conn.cursor() as cur: + # The first statement in a transaction can be retried transparently on + # the server, so we need to add a dummy statement so that our + # force_retry() statement isn't the first one. + cur.execute('SELECT now()') + cur.execute("SELECT crdb_internal.force_retry('1s'::INTERVAL)") + logging.debug("test_retry_loop(): status message: {}".format(cur.statusmessage)) + + +def transfer_funds(conn, frm, to, amount): + with conn.cursor() as cur: + + # Check the current balance. + cur.execute("SELECT balance FROM accounts WHERE id = " + str(frm)) + from_balance = cur.fetchone()[0] + if from_balance < amount: + err_msg = "Insufficient funds in account {}: have {}, need {}".format(frm, from_balance, amount) + raise RuntimeError(err_msg) + + # Perform the transfer. + cur.execute("UPDATE accounts SET balance = balance - %s WHERE id = %s", + (amount, frm)) + cur.execute("UPDATE accounts SET balance = balance + %s WHERE id = %s", + (amount, to)) + conn.commit() + logging.debug("transfer_funds(): status message: {}".format(cur.statusmessage)) + + +def main(): + + conn = psycopg2.connect( + database='bank', + user='maxroach', + sslmode='require', + sslrootcert='certs/ca.crt', + sslkey='certs/client.maxroach.key', + sslcert='certs/client.maxroach.crt', + port=26257, + host='localhost' + ) + + # Uncomment the below to turn on logging to the console. This was useful + # when testing transaction retry handling. It is not necessary for + # production code. + # log_level = getattr(logging, 'DEBUG', None) + # logging.basicConfig(level=log_level) + + create_accounts(conn) + + print_balances(conn) + + amount = 100 + fromId = 1 + toId = 2 + + try: + run_transaction(conn, lambda conn: transfer_funds(conn, fromId, toId, amount)) + + # The function below is used to test the transaction retry logic. It + # can be deleted from production code. + # run_transaction(conn, lambda conn: test_retry_loop(conn)) + except ValueError as ve: + # Below, we print the error and continue on so this example is easy to + # run (and run, and run...). In real code you should handle this error + # and any others thrown by the database interaction. + logging.debug("run_transaction(conn, op) failed: {}".format(ve)) + pass + + print_balances(conn) + + delete_accounts(conn) + + # Close communication with the database. + conn.close() + + +if __name__ == '__main__': + main() diff --git a/_includes/v20.2/app/basic-sample.rb b/_includes/v20.2/app/basic-sample.rb new file mode 100644 index 00000000000..93f0dc3d20c --- /dev/null +++ b/_includes/v20.2/app/basic-sample.rb @@ -0,0 +1,31 @@ +# Import the driver. +require 'pg' + +# Connect to the "bank" database. +conn = PG.connect( + user: 'maxroach', + dbname: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'require', + sslrootcert: 'certs/ca.crt', + sslkey:'certs/client.maxroach.key', + sslcert:'certs/client.maxroach.crt' +) + +# Create the "accounts" table. +conn.exec('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)') + +# Insert two rows into the "accounts" table. +conn.exec('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)') + +# Print out the balances. +puts 'Initial balances:' +conn.exec('SELECT id, balance FROM accounts') do |res| + res.each do |row| + puts row + end +end + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/basic-sample.rs b/_includes/v20.2/app/basic-sample.rs new file mode 100644 index 00000000000..4a078991cd8 --- /dev/null +++ b/_includes/v20.2/app/basic-sample.rs @@ -0,0 +1,45 @@ +use openssl::error::ErrorStack; +use openssl::ssl::{SslConnector, SslFiletype, SslMethod}; +use postgres::Client; +use postgres_openssl::MakeTlsConnector; + +fn ssl_config() -> Result { + let mut builder = SslConnector::builder(SslMethod::tls())?; + builder.set_ca_file("certs/ca.crt")?; + builder.set_certificate_chain_file("certs/client.maxroach.crt")?; + builder.set_private_key_file("certs/client.maxroach.key", SslFiletype::PEM)?; + Ok(MakeTlsConnector::new(builder.build())) +} + +fn main() { + let connector = ssl_config().unwrap(); + let mut client = + Client::connect("postgresql://maxroach@localhost:26257/bank", connector).unwrap(); + + // Create the "accounts" table. + client + .execute( + "CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", + &[], + ) + .unwrap(); + + // Insert two rows into the "accounts" table. + client + .execute( + "INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)", + &[], + ) + .unwrap(); + + // Print out the balances. + println!("Initial balances:"); + for row in &client + .query("SELECT id, balance FROM accounts", &[]) + .unwrap() + { + let id: i64 = row.get(0); + let balance: i64 = row.get(1); + println!("{} {}", id, balance); + } +} diff --git a/_includes/v20.2/app/before-you-begin.md b/_includes/v20.2/app/before-you-begin.md new file mode 100644 index 00000000000..dfb97226414 --- /dev/null +++ b/_includes/v20.2/app/before-you-begin.md @@ -0,0 +1,8 @@ +1. [Install CockroachDB](install-cockroachdb.html). +2. Start up a [secure](secure-a-cluster.html) or [insecure](start-a-local-cluster.html) local cluster. +3. Choose the instructions that correspond to whether your cluster is secure or insecure: + +
+ + +
diff --git a/_includes/v20.2/app/create-maxroach-user-and-bank-database.md b/_includes/v20.2/app/create-maxroach-user-and-bank-database.md new file mode 100644 index 00000000000..4d5b4626013 --- /dev/null +++ b/_includes/v20.2/app/create-maxroach-user-and-bank-database.md @@ -0,0 +1,32 @@ +Start the [built-in SQL shell](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs +~~~ + +In the SQL shell, issue the following statements to create the `maxroach` user and `bank` database: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER IF NOT EXISTS maxroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +Give the `maxroach` user the necessary permissions: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE bank TO maxroach; +~~~ + +Exit the SQL shell: + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ diff --git a/_includes/v20.2/app/django-basic-sample/models.py b/_includes/v20.2/app/django-basic-sample/models.py new file mode 100644 index 00000000000..6068f8bbb8e --- /dev/null +++ b/_includes/v20.2/app/django-basic-sample/models.py @@ -0,0 +1,17 @@ +from django.db import models + +class Customers(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=250) + +class Products(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=250) + price = models.DecimalField(max_digits=18, decimal_places=2) + +class Orders(models.Model): + id = models.AutoField(primary_key=True) + subtotal = models.DecimalField(max_digits=18, decimal_places=2) + customer = models.ForeignKey(Customers, on_delete=models.CASCADE, null=True) + product = models.ManyToManyField(Products) + diff --git a/_includes/v20.2/app/django-basic-sample/settings.py b/_includes/v20.2/app/django-basic-sample/settings.py new file mode 100644 index 00000000000..c94721d61e5 --- /dev/null +++ b/_includes/v20.2/app/django-basic-sample/settings.py @@ -0,0 +1,125 @@ +""" +Django settings for myproject project. + +Generated by 'django-admin startproject' using Django 3.0. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'spl=g73)8-)ja%x*k1eje4d#&24#t)zao^s$6vc1rdk(e3t!e(' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['0.0.0.0'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'myproject', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'myproject.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'myproject.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django_cockroachdb', + 'NAME': 'bank', + 'USER': 'django', + 'PASSWORD': 'password', + 'HOST': 'localhost', + 'PORT': '26257', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/_includes/v20.2/app/django-basic-sample/urls.py b/_includes/v20.2/app/django-basic-sample/urls.py new file mode 100644 index 00000000000..9550d713ffa --- /dev/null +++ b/_includes/v20.2/app/django-basic-sample/urls.py @@ -0,0 +1,20 @@ +from django.contrib import admin +from django.urls import path + +from .views import CustomersView, OrdersView, PingView, ProductView + +urlpatterns = [ + path('admin/', admin.site.urls), + + path('ping/', PingView.as_view()), + + # Endpoints for customers URL. + path('customer/', CustomersView.as_view(), name='customers'), + path('customer//', CustomersView.as_view(), name='customers'), + + # Endpoints for customers URL. + path('product/', ProductView.as_view(), name='product'), + path('product//', ProductView.as_view(), name='product'), + + path('order/', OrdersView.as_view(), name='order'), +] diff --git a/_includes/v20.2/app/django-basic-sample/views.py b/_includes/v20.2/app/django-basic-sample/views.py new file mode 100644 index 00000000000..78143916ee8 --- /dev/null +++ b/_includes/v20.2/app/django-basic-sample/views.py @@ -0,0 +1,107 @@ +from django.http import JsonResponse, HttpResponse +from django.utils.decorators import method_decorator +from django.views.generic import View +from django.views.decorators.csrf import csrf_exempt +from django.db import Error, IntegrityError +from django.db.transaction import atomic + +import json +import sys +import time + +from .models import * + +# Warning: Do not use retry_on_exception in an inner nested transaction. +def retry_on_exception(num_retries=3, on_failure=HttpResponse(status=500), delay_=0.5, backoff_=1.5): + def retry(view): + def wrapper(*args, **kwargs): + delay = delay_ + for i in range(num_retries): + try: + return view(*args, **kwargs) + except IntegrityError as ex: + if i == num_retries - 1: + return on_failure + elif getattr(ex.__cause__, 'pgcode', '') == errorcodes.SERIALIZATION_FAILURE: + time.sleep(delay) + delay *= backoff_ + except Error as ex: + return on_failure + return wrapper + return retry + +class PingView(View): + def get(self, request, *args, **kwargs): + return HttpResponse("python/django", status=200) + +@method_decorator(csrf_exempt, name='dispatch') +class CustomersView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + customers = list(Customers.objects.values()) + else: + customers = list(Customers.objects.filter(id=id).values()) + return JsonResponse(customers, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + name = form_data['name'] + c = Customers(name=name) + c.save() + return HttpResponse(status=200) + + @retry_on_exception(3) + @atomic + def delete(self, request, id=None, *args, **kwargs): + if id is None: + return HttpResponse(status=404) + Customers.objects.filter(id=id).delete() + return HttpResponse(status=200) + + # The PUT method is shadowed by the POST method, so there doesn't seem + # to be a reason to include it. + +@method_decorator(csrf_exempt, name='dispatch') +class ProductView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + products = list(Products.objects.values()) + else: + products = list(Products.objects.filter(id=id).values()) + return JsonResponse(products, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + name, price = form_data['name'], form_data['price'] + p = Products(name=name, price=price) + p.save() + return HttpResponse(status=200) + + # The REST API outlined in the github does not say that /product/ needs + # a PUT and DELETE method + +@method_decorator(csrf_exempt, name='dispatch') +class OrdersView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + orders = list(Orders.objects.values()) + else: + orders = list(Orders.objects.filter(id=id).values()) + return JsonResponse(orders, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + c = Customers.objects.get(id=form_data['customer']['id']) + o = Orders(subtotal=form_data['subtotal'], customer=c) + o.save() + for p in form_data['products']: + p = Products.objects.get(id=p['id']) + o.product.add(p) + o.save() + return HttpResponse(status=200) diff --git a/_includes/v20.2/app/for-a-complete-example-go.md b/_includes/v20.2/app/for-a-complete-example-go.md new file mode 100644 index 00000000000..e0144fb1f4f --- /dev/null +++ b/_includes/v20.2/app/for-a-complete-example-go.md @@ -0,0 +1,4 @@ +For complete examples, see: + +- [Build a Go App with CockroachDB](build-a-go-app-with-cockroachdb.html) (pq) +- [Build a Go App with CockroachDB and GORM](build-a-go-app-with-cockroachdb.html) diff --git a/_includes/v20.2/app/for-a-complete-example-java.md b/_includes/v20.2/app/for-a-complete-example-java.md new file mode 100644 index 00000000000..b4c63135ae0 --- /dev/null +++ b/_includes/v20.2/app/for-a-complete-example-java.md @@ -0,0 +1,4 @@ +For complete examples, see: + +- [Build a Java App with CockroachDB](build-a-java-app-with-cockroachdb.html) (JDBC) +- [Build a Java App with CockroachDB and Hibernate](build-a-java-app-with-cockroachdb-hibernate.html) diff --git a/_includes/v20.2/app/for-a-complete-example-python.md b/_includes/v20.2/app/for-a-complete-example-python.md new file mode 100644 index 00000000000..432aa82a1d6 --- /dev/null +++ b/_includes/v20.2/app/for-a-complete-example-python.md @@ -0,0 +1,6 @@ +For complete examples, see: + +- [Build a Python App with CockroachDB](build-a-python-app-with-cockroachdb.html) (psycopg2) +- [Build a Python App with CockroachDB and SQLAlchemy](build-a-python-app-with-cockroachdb-sqlalchemy.html) +- [Build a Python App with CockroachDB and Django](build-a-python-app-with-cockroachdb-django.html) +- [Build a Python App with CockroachDB and PonyORM](build-a-python-app-with-cockroachdb-pony.html) diff --git a/_includes/v20.2/app/gorm-basic-sample.go b/_includes/v20.2/app/gorm-basic-sample.go new file mode 100644 index 00000000000..d18948b80b2 --- /dev/null +++ b/_includes/v20.2/app/gorm-basic-sample.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "log" + + // Import GORM-related packages. + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" +) + +// Account is our model, which corresponds to the "accounts" database table. +type Account struct { + ID int `gorm:"primary_key"` + Balance int +} + +func main() { + // Connect to the "bank" database as the "maxroach" user. + const addr = "postgresql://maxroach@localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key&sslcert=certs/client.maxroach.crt" + db, err := gorm.Open("postgres", addr) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Automatically create the "accounts" table based on the Account model. + db.AutoMigrate(&Account{}) + + // Insert two rows into the "accounts" table. + db.Create(&Account{ID: 1, Balance: 1000}) + db.Create(&Account{ID: 2, Balance: 250}) + + // Print out the balances. + var accounts []Account + db.Find(&accounts) + fmt.Println("Initial balances:") + for _, account := range accounts { + fmt.Printf("%d %d\n", account.ID, account.Balance) + } +} diff --git a/_includes/v20.2/app/gorm-sample.go b/_includes/v20.2/app/gorm-sample.go new file mode 100644 index 00000000000..a49089c5509 --- /dev/null +++ b/_includes/v20.2/app/gorm-sample.go @@ -0,0 +1,206 @@ +package main + +import ( + "fmt" + "log" + "math" + "math/rand" + "time" + + // Import GORM-related packages. + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" + + // Necessary in order to check for transaction retry error codes. + "github.com/lib/pq" +) + +// Account is our model, which corresponds to the "accounts" database +// table. +type Account struct { + ID int `gorm:"primary_key"` + Balance int +} + +// Functions of type `txnFunc` are passed as arguments to our +// `runTransaction` wrapper that handles transaction retries for us +// (see implementation below). +type txnFunc func(*gorm.DB) error + +// This function is used for testing the transaction retry loop. It +// can be deleted from production code. +var forceRetryLoop txnFunc = func(db *gorm.DB) error { + + // The first statement in a transaction can be retried transparently + // on the server, so we need to add a dummy statement so that our + // force_retry statement isn't the first one. + if err := db.Exec("SELECT now()").Error; err != nil { + return err + } + // Used to force a transaction retry. + if err := db.Exec("SELECT crdb_internal.force_retry('1s'::INTERVAL)").Error; err != nil { + return err + } + return nil +} + +func transferFunds(db *gorm.DB, fromID int, toID int, amount int) error { + var fromAccount Account + var toAccount Account + + db.First(&fromAccount, fromID) + db.First(&toAccount, toID) + + if fromAccount.Balance < amount { + return fmt.Errorf("account %d balance %d is lower than transfer amount %d", fromAccount.ID, fromAccount.Balance, amount) + } + + fromAccount.Balance -= amount + toAccount.Balance += amount + + if err := db.Save(&fromAccount).Error; err != nil { + return err + } + if err := db.Save(&toAccount).Error; err != nil { + return err + } + return nil +} + +func main() { + // Connect to the "bank" database as the "maxroach" user. + const addr = "postgresql://maxroach@localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key&sslcert=certs/client.maxroach.crt" + db, err := gorm.Open("postgres", addr) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Set to `true` and GORM will print out all DB queries. + db.LogMode(false) + + // Automatically create the "accounts" table based on the Account + // model. + db.AutoMigrate(&Account{}) + + // Insert two rows into the "accounts" table. + var fromID = 1 + var toID = 2 + db.Create(&Account{ID: fromID, Balance: 1000}) + db.Create(&Account{ID: toID, Balance: 250}) + + // The sequence of steps in this section is: + // 1. Print account balances. + // 2. Set up some Accounts and transfer funds between them inside + // a transaction. + // 3. Print account balances again to verify the transfer occurred. + + // Print balances before transfer. + printBalances(db) + + // The amount to be transferred between the accounts. + var amount = 100 + + // Transfer funds between accounts. To handle potential + // transaction retry errors, we wrap the call to `transferFunds` + // in `runTransaction`, a wrapper which implements a retry loop + // with exponential backoff around our access to the database (see + // the implementation for details). + if err := runTransaction(db, + func(*gorm.DB) error { + return transferFunds(db, fromID, toID, amount) + }, + ); err != nil { + // If the error is returned, it's either: + // 1. Not a transaction retry error, i.e., some other kind + // of database error that you should handle here. + // 2. A transaction retry error that has occurred more than + // N times (defined by the `maxRetries` variable inside + // `runTransaction`), in which case you will need to figure + // out why your database access is resulting in so much + // contention (see 'Understanding and avoiding transaction + // contention': + // https://www.cockroachlabs.com/docs/stable/performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) + fmt.Println(err) + } + + // Print balances after transfer to ensure that it worked. + printBalances(db) + + // Delete accounts so we can start fresh when we want to run this + // program again. + deleteAccounts(db) +} + +// Wrapper for a transaction. This automatically re-calls `fn` with +// the open transaction as an argument as long as the database server +// asks for the transaction to be retried. +func runTransaction(db *gorm.DB, fn txnFunc) error { + var maxRetries = 3 + for retries := 0; retries <= maxRetries; retries++ { + if retries == maxRetries { + return fmt.Errorf("hit max of %d retries, aborting", retries) + } + txn := db.Begin() + if err := fn(txn); err != nil { + // We need to cast GORM's db.Error to *pq.Error so we can + // detect the Postgres transaction retry error code and + // handle retries appropriately. + pqErr := err.(*pq.Error) + if pqErr.Code == "40001" { + // Since this is a transaction retry error, we + // ROLLBACK the transaction and sleep a little before + // trying again. Each time through the loop we sleep + // for a little longer than the last time + // (A.K.A. exponential backoff). + txn.Rollback() + var sleepMs = math.Pow(2, float64(retries)) * 100 * (rand.Float64() + 0.5) + fmt.Printf("Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMs) + time.Sleep(time.Millisecond * time.Duration(sleepMs)) + } else { + // If it's not a retry error, it's some other sort of + // DB interaction error that needs to be handled by + // the caller. + return err + } + } else { + // All went well, so we try to commit and break out of the + // retry loop if possible. + if err := txn.Commit().Error; err != nil { + pqErr := err.(*pq.Error) + if pqErr.Code == "40001" { + // However, our attempt to COMMIT could also + // result in a retry error, in which case we + // continue back through the loop and try again. + continue + } else { + // If it's not a retry error, it's some other sort + // of DB interaction error that needs to be + // handled by the caller. + return err + } + } + break + } + } + return nil +} + +func printBalances(db *gorm.DB) { + var accounts []Account + db.Find(&accounts) + fmt.Printf("Balance at '%s':\n", time.Now()) + for _, account := range accounts { + fmt.Printf("%d %d\n", account.ID, account.Balance) + } +} + +func deleteAccounts(db *gorm.DB) error { + // Used to tear down the accounts table so we can re-run this + // program. + err := db.Exec("DELETE from accounts where ID > 0").Error + if err != nil { + return err + } + return nil +} diff --git a/_includes/v20.2/app/hibernate-basic-sample/Sample.java b/_includes/v20.2/app/hibernate-basic-sample/Sample.java new file mode 100644 index 00000000000..58d28f37a4b --- /dev/null +++ b/_includes/v20.2/app/hibernate-basic-sample/Sample.java @@ -0,0 +1,236 @@ +package com.cockroachlabs; + +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.JDBCException; +import org.hibernate.cfg.Configuration; + +import java.util.*; +import java.util.function.Function; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +public class Sample { + + private static final Random RAND = new Random(); + private static final boolean FORCE_RETRY = false; + private static final String RETRY_SQL_STATE = "40001"; + private static final int MAX_ATTEMPT_COUNT = 6; + + // Account is our model, which corresponds to the "accounts" database table. + @Entity + @Table(name="accounts") + public static class Account { + @Id + @Column(name="id") + public long id; + + public long getId() { + return id; + } + + @Column(name="balance") + public long balance; + public long getBalance() { + return balance; + } + public void setBalance(long newBalance) { + this.balance = newBalance; + } + + // Convenience constructor. + public Account(int id, int balance) { + this.id = id; + this.balance = balance; + } + + // Hibernate needs a default (no-arg) constructor to create model objects. + public Account() {} + } + + private static Function addAccounts() throws JDBCException{ + Function f = s -> { + long rv = 0; + try { + s.save(new Account(1, 1000)); + s.save(new Account(2, 250)); + s.save(new Account(3, 314159)); + rv = 1; + System.out.printf("APP: addAccounts() --> %d\n", rv); + } catch (JDBCException e) { + throw e; + } + return rv; + }; + return f; + } + + private static Function transferFunds(long fromId, long toId, long amount) throws JDBCException{ + Function f = s -> { + long rv = 0; + try { + Account fromAccount = (Account) s.get(Account.class, fromId); + Account toAccount = (Account) s.get(Account.class, toId); + if (!(amount > fromAccount.getBalance())) { + fromAccount.balance -= amount; + toAccount.balance += amount; + s.save(fromAccount); + s.save(toAccount); + rv = amount; + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d\n", fromId, toId, amount, rv); + } + } catch (JDBCException e) { + throw e; + } + return rv; + }; + return f; + } + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // intended for production code. + private static Function forceRetryLogic() throws JDBCException { + Function f = s -> { + long rv = -1; + try { + System.out.printf("APP: testRetryLogic: BEFORE EXCEPTION\n"); + s.createNativeQuery("SELECT crdb_internal.force_retry('1s')").executeUpdate(); + } catch (JDBCException e) { + System.out.printf("APP: testRetryLogic: AFTER EXCEPTION\n"); + throw e; + } + return rv; + }; + return f; + } + + private static Function getAccountBalance(long id) throws JDBCException{ + Function f = s -> { + long balance; + try { + Account account = s.get(Account.class, id); + balance = account.getBalance(); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", id, balance); + } catch (JDBCException e) { + throw e; + } + return balance; + }; + return f; + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic so we don't have to duplicate it in + // various places. + private static long runTransaction(Session session, Function fn) { + long rv = 0; + int attemptCount = 0; + + while (attemptCount < MAX_ATTEMPT_COUNT) { + attemptCount++; + + if (attemptCount > 1) { + System.out.printf("APP: Entering retry loop again, iteration %d\n", attemptCount); + } + + Transaction txn = session.beginTransaction(); + System.out.printf("APP: BEGIN;\n"); + + if (attemptCount == MAX_ATTEMPT_COUNT) { + String err = String.format("hit max of %s attempts, aborting", MAX_ATTEMPT_COUNT); + throw new RuntimeException(err); + } + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryLogic()'. + if (FORCE_RETRY) { + session.createNativeQuery("SELECT now()").list(); + } + + try { + rv = fn.apply(session); + if (rv != -1) { + txn.commit(); + System.out.printf("APP: COMMIT;\n"); + break; + } + } catch (JDBCException e) { + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + // Since this is a transaction retry error, we + // roll back the transaction and sleep a little + // before trying again. Each time through the + // loop we sleep for a little longer than the last + // time (A.K.A. exponential backoff). + System.out.printf("APP: retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", e.getSQLState(), e.getMessage(), attemptCount); + System.out.printf("APP: ROLLBACK;\n"); + txn.rollback(); + int sleepMillis = (int)(Math.pow(2, attemptCount) * 100) + RAND.nextInt(100); + System.out.printf("APP: Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMillis); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException ignored) { + // no-op + } + rv = -1; + } else { + throw e; + } + } + } + return rv; + } + + public static void main(String[] args) { + // Create a SessionFactory based on our hibernate.cfg.xml configuration + // file, which defines how to connect to the database. + SessionFactory sessionFactory = + new Configuration() + .configure("hibernate.cfg.xml") + .addAnnotatedClass(Account.class) + .buildSessionFactory(); + + try (Session session = sessionFactory.openSession()) { + long fromAccountId = 1; + long toAccountId = 2; + long transferAmount = 100; + + if (FORCE_RETRY) { + System.out.printf("APP: About to test retry logic in 'runTransaction'\n"); + runTransaction(session, forceRetryLogic()); + } else { + + runTransaction(session, addAccounts()); + long fromBalance = runTransaction(session, getAccountBalance(fromAccountId)); + long toBalance = runTransaction(session, getAccountBalance(toAccountId)); + if (fromBalance != -1 && toBalance != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalance); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalance); + } + + // Transfer $100 from account 1 to account 2 + long transferResult = runTransaction(session, transferFunds(fromAccountId, toAccountId, transferAmount)); + if (transferResult != -1) { + // Success! + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d \n", fromAccountId, toAccountId, transferAmount, transferResult); + + long fromBalanceAfter = runTransaction(session, getAccountBalance(fromAccountId)); + long toBalanceAfter = runTransaction(session, getAccountBalance(toAccountId)); + if (fromBalanceAfter != -1 && toBalanceAfter != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalanceAfter); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalanceAfter); + } + } + } + } finally { + sessionFactory.close(); + } + } +} diff --git a/_includes/v20.2/app/hibernate-basic-sample/build.gradle b/_includes/v20.2/app/hibernate-basic-sample/build.gradle new file mode 100644 index 00000000000..36f33d73fe6 --- /dev/null +++ b/_includes/v20.2/app/hibernate-basic-sample/build.gradle @@ -0,0 +1,16 @@ +group 'com.cockroachlabs' +version '1.0' + +apply plugin: 'java' +apply plugin: 'application' + +mainClassName = 'com.cockroachlabs.Sample' + +repositories { + mavenCentral() +} + +dependencies { + compile 'org.hibernate:hibernate-core:5.2.4.Final' + compile 'org.postgresql:postgresql:42.2.2.jre7' +} diff --git a/_includes/v20.2/app/hibernate-basic-sample/hibernate-basic-sample.tgz b/_includes/v20.2/app/hibernate-basic-sample/hibernate-basic-sample.tgz new file mode 100644 index 00000000000..3e729bf439e Binary files /dev/null and b/_includes/v20.2/app/hibernate-basic-sample/hibernate-basic-sample.tgz differ diff --git a/_includes/v20.2/app/hibernate-basic-sample/hibernate.cfg.xml b/_includes/v20.2/app/hibernate-basic-sample/hibernate.cfg.xml new file mode 100644 index 00000000000..454a4950ad0 --- /dev/null +++ b/_includes/v20.2/app/hibernate-basic-sample/hibernate.cfg.xml @@ -0,0 +1,21 @@ + + + + + + + org.postgresql.Driver + org.hibernate.dialect.PostgreSQL95Dialect + + maxroach + + + create + + + true + true + + diff --git a/_includes/v20.2/app/insecure/BasicExample.java b/_includes/v20.2/app/insecure/BasicExample.java new file mode 100644 index 00000000000..de86e98d93f --- /dev/null +++ b/_includes/v20.2/app/insecure/BasicExample.java @@ -0,0 +1,433 @@ +import java.util.*; +import java.time.*; +import java.sql.*; +import javax.sql.DataSource; + +import org.postgresql.ds.PGSimpleDataSource; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicExample.java && java BasicExample + + To build the javadoc: + + $ javadoc -package -cp .:./path/to/postgresql.jar BasicExample.java + + At a high level, this code consists of two classes: + + 1. BasicExample, which is where the application logic lives. + + 2. BasicExampleDAO, which is used by the application to access the + data store. + +*/ + +public class BasicExample { + + public static void main(String[] args) { + + // Configure the database connection. + PGSimpleDataSource ds = new PGSimpleDataSource(); + ds.setServerName("localhost"); + ds.setPortNumber(26257); + ds.setDatabaseName("bank"); + ds.setUser("maxroach"); + ds.setPassword(null); + ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string + ds.setApplicationName("BasicExample"); + + // Create DAO. + BasicExampleDAO dao = new BasicExampleDAO(ds); + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // necessary in production code. + dao.testRetryHandling(); + + // Set up the 'accounts' table. + dao.createAccounts(); + + // Insert a few accounts "by hand", using INSERTs on the backend. + Map balances = new HashMap(); + balances.put("1", "1000"); + balances.put("2", "250"); + int updatedAccounts = dao.updateAccounts(balances); + System.out.printf("BasicExampleDAO.updateAccounts:\n => %s total updated accounts\n", updatedAccounts); + + // How much money is in these accounts? + int balance1 = dao.getAccountBalance(1); + int balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Transfer $100 from account 1 to account 2 + int fromAccount = 1; + int toAccount = 2; + int transferAmount = 100; + int transferredAccounts = dao.transferFunds(fromAccount, toAccount, transferAmount); + if (transferredAccounts != -1) { + System.out.printf("BasicExampleDAO.transferFunds:\n => $%s transferred between accounts %s and %s, %s rows updated\n", transferAmount, fromAccount, toAccount, transferredAccounts); + } + + balance1 = dao.getAccountBalance(1); + balance2 = dao.getAccountBalance(2); + System.out.printf("main:\n => Account balances at time '%s':\n ID %s => $%s\n ID %s => $%s\n", LocalTime.now(), 1, balance1, 2, balance2); + + // Bulk insertion example using JDBC's batching support. + int totalRowsInserted = dao.bulkInsertRandomAccountData(); + System.out.printf("\nBasicExampleDAO.bulkInsertRandomAccountData:\n => finished, %s total rows inserted\n", totalRowsInserted); + + // Print out 10 account values. + int accountsRead = dao.readAccounts(10); + + // Drop the 'accounts' table so this code can be run again. + dao.tearDown(); + } +} + +/** + * Data access object used by 'BasicExample'. Abstraction over some + * common CockroachDB operations, including: + * + * - Auto-handling transaction retries in the 'runSQL' method + * + * - Example of bulk inserts in the 'bulkInsertRandomAccountData' + * method + */ + +class BasicExampleDAO { + + private static final int MAX_RETRY_COUNT = 3; + private static final String SAVEPOINT_NAME = "cockroach_restart"; + private static final String RETRY_SQL_STATE = "40001"; + private static final boolean FORCE_RETRY = false; + + private final DataSource ds; + + BasicExampleDAO(DataSource ds) { + this.ds = ds; + } + + /** + Used to test the retry logic in 'runSQL'. It is not necessary + in production code. + */ + void testRetryHandling() { + if (this.FORCE_RETRY) { + runSQL("SELECT crdb_internal.force_retry('1s':::INTERVAL)"); + } + } + + /** + * Run SQL code in a way that automatically handles the + * transaction retry logic so we don't have to duplicate it in + * various places. + * + * @param sqlCode a String containing the SQL code you want to + * execute. Can have placeholders, e.g., "INSERT INTO accounts + * (id, balance) VALUES (?, ?)". + * + * @param args String Varargs to fill in the SQL code's + * placeholders. + * @return Integer Number of rows updated, or -1 if an error is thrown. + */ + public Integer runSQL(String sqlCode, String... args) { + + // This block is only used to emit class and method names in + // the program output. It is not necessary in production + // code. + StackTraceElement[] stacktrace = Thread.currentThread().getStackTrace(); + StackTraceElement elem = stacktrace[2]; + String callerClass = elem.getClassName(); + String callerMethod = elem.getMethodName(); + + int rv = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // automatically issue transaction retries. + connection.setAutoCommit(false); + + int retryCount = 0; + + while (retryCount < MAX_RETRY_COUNT) { + + Savepoint sp = connection.setSavepoint(SAVEPOINT_NAME); + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryHandling()'. + if (FORCE_RETRY) { + forceRetry(connection); // SELECT 1 + } + + try (PreparedStatement pstmt = connection.prepareStatement(sqlCode)) { + + // Loop over the args and insert them into the + // prepared statement based on their types. In + // this simple example we classify the argument + // types as "integers" and "everything else" + // (a.k.a. strings). + for (int i=0; i %10s\n", name, val); + } + } + } + } else { + int updateCount = pstmt.getUpdateCount(); + rv += updateCount; + + // This printed output is for debugging and/or demonstration + // purposes only. It would not be necessary in production code. + System.out.printf("\n%s.%s:\n '%s'\n", callerClass, callerMethod, pstmt); + } + + connection.releaseSavepoint(sp); + connection.commit(); + break; + + } catch (SQLException e) { + + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + System.out.printf("retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", + e.getSQLState(), e.getMessage(), retryCount); + connection.rollback(sp); + retryCount++; + rv = -1; + } else { + rv = -1; + throw e; + } + } + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.runSQL ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + rv = -1; + } + + return rv; + } + + /** + * Helper method called by 'testRetryHandling'. It simply issues + * a "SELECT 1" inside the transaction to force a retry. This is + * necessary to take the connection's session out of the AutoRetry + * state, since otherwise the other statements in the session will + * be retried automatically, and the client (us) will not see a + * retry error. Note that this information is taken from the + * following test: + * https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/logictest/testdata/logic_test/manual_retry + * + * @param connection Connection + */ + private void forceRetry(Connection connection) throws SQLException { + try (PreparedStatement statement = connection.prepareStatement("SELECT 1")){ + statement.executeQuery(); + } + } + + /** + * Creates a fresh, empty accounts table in the database. + */ + public void createAccounts() { + runSQL("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))"); + }; + + /** + * Update accounts by passing in a Map of (ID, Balance) pairs. + * + * @param accounts (Map) + * @return The number of updated accounts (int) + */ + public int updateAccounts(Map accounts) { + int rows = 0; + for (Map.Entry account : accounts.entrySet()) { + + String k = account.getKey(); + String v = account.getValue(); + + String[] args = {k, v}; + rows += runSQL("INSERT INTO accounts (id, balance) VALUES (?, ?)", args); + } + return rows; + } + + /** + * Transfer funds between one account and another. Handles + * transaction retries in case of conflict automatically on the + * backend. + * @param fromId (int) + * @param toId (int) + * @param amount (int) + * @return The number of updated accounts (int) + */ + public int transferFunds(int fromId, int toId, int amount) { + String sFromId = Integer.toString(fromId); + String sToId = Integer.toString(toId); + String sAmount = Integer.toString(amount); + + // We have omitted explicit BEGIN/COMMIT statements for + // brevity. Individual statements are treated as implicit + // transactions by CockroachDB (see + // https://www.cockroachlabs.com/docs/stable/transactions.html#individual-statements). + + String sqlCode = "UPSERT INTO accounts (id, balance) VALUES" + + "(?, ((SELECT balance FROM accounts WHERE id = ?) - ?))," + + "(?, ((SELECT balance FROM accounts WHERE id = ?) + ?))"; + + return runSQL(sqlCode, sFromId, sFromId, sAmount, sToId, sToId, sAmount); + } + + /** + * Get the account balance for one account. + * + * We skip using the retry logic in 'runSQL()' here for the + * following reasons: + * + * 1. Since this is a single read ("SELECT"), we don't expect any + * transaction conflicts to handle + * + * 2. We need to return the balance as an integer + * + * @param id (int) + * @return balance (int) + */ + public int getAccountBalance(int id) { + int balance = 0; + + try (Connection connection = ds.getConnection()) { + + // Check the current balance. + ResultSet res = connection.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + id); + if(!res.next()) { + System.out.printf("No users in the table with id %i", id); + } else { + balance = res.getInt("balance"); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.getAccountBalance ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + + return balance; + } + + /** + * Insert randomized account data (ID, balance) using the JDBC + * fast path for bulk inserts. The fastest way to get data into + * CockroachDB is the IMPORT statement. However, if you must bulk + * ingest from the application using INSERT statements, the best + * option is the method shown here. It will require the following: + * + * 1. Add `rewriteBatchedInserts=true` to your JDBC connection + * settings (see the connection info in 'BasicExample.main'). + * + * 2. Inserting in batches of 128 rows, as used inside this method + * (see BATCH_SIZE), since the PGJDBC driver's logic works best + * with powers of two, such that a batch of size 128 can be 6x + * faster than a batch of size 250. + * @return The number of new accounts inserted (int) + */ + public int bulkInsertRandomAccountData() { + + Random random = new Random(); + int BATCH_SIZE = 128; + int totalNewAccounts = 0; + + try (Connection connection = ds.getConnection()) { + + // We're managing the commit lifecycle ourselves so we can + // control the size of our batch inserts. + connection.setAutoCommit(false); + + // In this example we are adding 500 rows to the database, + // but it could be any number. What's important is that + // the batch size is 128. + try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); + } + connection.commit(); + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + } catch (SQLException e) { + System.out.printf("BasicExampleDAO.bulkInsertRandomAccountData ERROR: { state => %s, cause => %s, message => %s }\n", + e.getSQLState(), e.getCause(), e.getMessage()); + } + return totalNewAccounts; + } + + /** + * Read out a subset of accounts from the data store. + * + * @param limit (int) + * @return Number of accounts read (int) + */ + public int readAccounts(int limit) { + return runSQL("SELECT id, balance FROM accounts LIMIT ?", Integer.toString(limit)); + } + + /** + * Perform any necessary cleanup of the data store so it can be + * used again. + */ + public void tearDown() { + runSQL("DROP TABLE accounts;"); + } +} diff --git a/_includes/v20.2/app/insecure/BasicSample.java b/_includes/v20.2/app/insecure/BasicSample.java new file mode 100644 index 00000000000..001d38feb48 --- /dev/null +++ b/_includes/v20.2/app/insecure/BasicSample.java @@ -0,0 +1,51 @@ +import java.sql.*; +import java.util.Properties; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac BasicSample.java && java BasicSample +*/ + +public class BasicSample { + public static void main(String[] args) + throws ClassNotFoundException, SQLException { + + // Load the Postgres JDBC driver. + Class.forName("org.postgresql.Driver"); + + // Connect to the "bank" database. + Properties props = new Properties(); + props.setProperty("user", "maxroach"); + props.setProperty("sslmode", "disable"); + + Connection db = DriverManager + .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); + + try { + // Create the "accounts" table. + db.createStatement() + .execute("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); + + // Insert two rows into the "accounts" table. + db.createStatement() + .execute("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); + + // Print out the balances. + System.out.println("Initial balances:"); + ResultSet res = db.createStatement() + .executeQuery("SELECT id, balance FROM accounts"); + while (res.next()) { + System.out.printf("\taccount %s: %s\n", + res.getInt("id"), + res.getInt("balance")); + } + } finally { + // Close the database connection. + db.close(); + } + } +} diff --git a/_includes/v20.2/app/insecure/TxnSample.java b/_includes/v20.2/app/insecure/TxnSample.java new file mode 100644 index 00000000000..11021ec0e71 --- /dev/null +++ b/_includes/v20.2/app/insecure/TxnSample.java @@ -0,0 +1,145 @@ +import java.sql.*; +import java.util.Properties; + +/* + Download the Postgres JDBC driver jar from https://jdbc.postgresql.org. + + Then, compile and run this example like so: + + $ export CLASSPATH=.:/path/to/postgresql.jar + $ javac TxnSample.java && java TxnSample +*/ + +// Ambiguous whether the transaction committed or not. +class AmbiguousCommitException extends SQLException{ + public AmbiguousCommitException(Throwable cause) { + super(cause); + } +} + +class InsufficientBalanceException extends Exception {} + +class AccountNotFoundException extends Exception { + public int account; + public AccountNotFoundException(int account) { + this.account = account; + } +} + +// A simple interface that provides a retryable lambda expression. +interface RetryableTransaction { + public void run(Connection conn) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException; +} + +public class TxnSample { + public static RetryableTransaction transferFunds(int from, int to, int amount) { + return new RetryableTransaction() { + public void run(Connection conn) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException { + + // Check the current balance. + ResultSet res = conn.createStatement() + .executeQuery("SELECT balance FROM accounts WHERE id = " + + from); + if(!res.next()) { + throw new AccountNotFoundException(from); + } + + int balance = res.getInt("balance"); + if(balance < from) { + throw new InsufficientBalanceException(); + } + + // Perform the transfer. + conn.createStatement() + .executeUpdate("UPDATE accounts SET balance = balance - " + + amount + " where id = " + from); + conn.createStatement() + .executeUpdate("UPDATE accounts SET balance = balance + " + + amount + " where id = " + to); + } + }; + } + + public static void retryTransaction(Connection conn, RetryableTransaction tx) + throws SQLException, InsufficientBalanceException, + AccountNotFoundException, AmbiguousCommitException { + + Savepoint sp = conn.setSavepoint("cockroach_restart"); + while(true) { + boolean releaseAttempted = false; + try { + tx.run(conn); + releaseAttempted = true; + conn.releaseSavepoint(sp); + } + catch(SQLException e) { + String sqlState = e.getSQLState(); + + // Check if the error code indicates a SERIALIZATION_FAILURE. + if(sqlState.equals("40001")) { + // Signal the database that we will attempt a retry. + conn.rollback(sp); + continue; + } else if(releaseAttempted) { + throw new AmbiguousCommitException(e); + } else { + throw e; + } + } + break; + } + conn.commit(); + } + + public static void main(String[] args) + throws ClassNotFoundException, SQLException { + + // Load the Postgres JDBC driver. + Class.forName("org.postgresql.Driver"); + + // Connect to the 'bank' database. + Properties props = new Properties(); + props.setProperty("user", "maxroach"); + props.setProperty("sslmode", "disable"); + + Connection db = DriverManager + .getConnection("jdbc:postgresql://127.0.0.1:26257/bank", props); + + + try { + // We need to turn off autocommit mode to allow for + // multi-statement transactions. + db.setAutoCommit(false); + + // Perform the transfer. This assumes the 'accounts' + // table has already been created in the database. + RetryableTransaction transfer = transferFunds(1, 2, 100); + retryTransaction(db, transfer); + + // Check balances after transfer. + db.setAutoCommit(true); + ResultSet res = db.createStatement() + .executeQuery("SELECT id, balance FROM accounts"); + while (res.next()) { + System.out.printf("\taccount %s: %s\n", res.getInt("id"), + res.getInt("balance")); + } + + } catch(InsufficientBalanceException e) { + System.out.println("Insufficient balance"); + } catch(AccountNotFoundException e) { + System.out.println("No users in the table with id " + e.account); + } catch(AmbiguousCommitException e) { + System.out.println("Ambiguous result encountered: " + e); + } catch(SQLException e) { + System.out.println("SQLException encountered:" + e); + } finally { + // Close the database connection. + db.close(); + } + } +} diff --git a/_includes/v20.2/app/insecure/activerecord-basic-sample.rb b/_includes/v20.2/app/insecure/activerecord-basic-sample.rb new file mode 100644 index 00000000000..601838ee789 --- /dev/null +++ b/_includes/v20.2/app/insecure/activerecord-basic-sample.rb @@ -0,0 +1,44 @@ +require 'active_record' +require 'activerecord-cockroachdb-adapter' +require 'pg' + +# Connect to CockroachDB through ActiveRecord. +# In Rails, this configuration would go in config/database.yml as usual. +ActiveRecord::Base.establish_connection( + adapter: 'cockroachdb', + username: 'maxroach', + database: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'disable' +) + +# Define the Account model. +# In Rails, this would go in app/models/ as usual. +class Account < ActiveRecord::Base + validates :id, presence: true + validates :balance, presence: true +end + +# Define a migration for the accounts table. +# In Rails, this would go in db/migrate/ as usual. +class Schema < ActiveRecord::Migration[5.0] + def change + create_table :accounts, force: true do |t| + t.integer :balance + end + end +end + +# Run the schema migration by hand. +# In Rails, this would be done via rake db:migrate as usual. +Schema.new.change() + +# Create two accounts, inserting two rows into the accounts table. +Account.create(id: 1, balance: 1000) +Account.create(id: 2, balance: 250) + +# Retrieve accounts and print out the balances +Account.all.each do |acct| + puts "#{acct.id} #{acct.balance}" +end diff --git a/_includes/v20.2/app/insecure/basic-sample.clj b/_includes/v20.2/app/insecure/basic-sample.clj new file mode 100644 index 00000000000..182b78b675e --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.clj @@ -0,0 +1,31 @@ +(ns test.test + (:require [clojure.java.jdbc :as j] + [test.util :as util])) + +;; Define the connection parameters to the cluster. +(def db-spec {:dbtype "postgresql" + :dbname "bank" + :host "localhost" + :port "26257" + :user "maxroach"}) + +(defn test-basic [] + ;; Connect to the cluster and run the code below with + ;; the connection object bound to 'conn'. + (j/with-db-connection [conn db-spec] + + ;; Insert two rows into the "accounts" table. + (j/insert! conn :accounts {:id 1 :balance 1000}) + (j/insert! conn :accounts {:id 2 :balance 250}) + + ;; Print out the balances. + (println "Initial balances:") + (->> (j/query conn ["SELECT id, balance FROM accounts"]) + (map println) + doall) + + )) + + +(defn -main [& args] + (test-basic)) diff --git a/_includes/v20.2/app/insecure/basic-sample.cpp b/_includes/v20.2/app/insecure/basic-sample.cpp new file mode 100644 index 00000000000..a06d84d1a25 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.cpp @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; + +int main() { + try { + // Connect to the "bank" database. + pqxx::connection c("postgresql://maxroach@localhost:26257/bank"); + + pqxx::nontransaction w(c); + + // Create the "accounts" table. + w.exec("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); + + // Insert two rows into the "accounts" table. + w.exec("INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); + + // Print out the balances. + cout << "Initial balances:" << endl; + pqxx::result r = w.exec("SELECT id, balance FROM accounts"); + for (auto row : r) { + cout << row[0].as() << ' ' << row[1].as() << endl; + } + + w.commit(); // Note this doesn't doesn't do anything + // for a nontransaction, but is still required. + } + catch (const exception &e) { + cerr << e.what() << endl; + return 1; + } + cout << "Success" << endl; + return 0; +} diff --git a/_includes/v20.2/app/insecure/basic-sample.cs b/_includes/v20.2/app/insecure/basic-sample.cs new file mode 100644 index 00000000000..b7cf8e1ff3f --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.cs @@ -0,0 +1,50 @@ +using System; +using System.Data; +using Npgsql; + +namespace Cockroach +{ + class MainClass + { + static void Main(string[] args) + { + var connStringBuilder = new NpgsqlConnectionStringBuilder(); + connStringBuilder.Host = "localhost"; + connStringBuilder.Port = 26257; + connStringBuilder.SslMode = SslMode.Disable; + connStringBuilder.Username = "maxroach"; + connStringBuilder.Database = "bank"; + Simple(connStringBuilder.ConnectionString); + } + + static void Simple(string connString) + { + using (var conn = new NpgsqlConnection(connString)) + { + conn.Open(); + + // Create the "accounts" table. + new NpgsqlCommand("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", conn).ExecuteNonQuery(); + + // Insert two rows into the "accounts" table. + using (var cmd = new NpgsqlCommand()) + { + cmd.Connection = conn; + cmd.CommandText = "UPSERT INTO accounts(id, balance) VALUES(@id1, @val1), (@id2, @val2)"; + cmd.Parameters.AddWithValue("id1", 1); + cmd.Parameters.AddWithValue("val1", 1000); + cmd.Parameters.AddWithValue("id2", 2); + cmd.Parameters.AddWithValue("val2", 250); + cmd.ExecuteNonQuery(); + } + + // Print out the balances. + System.Console.WriteLine("Initial balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + } + } + } +} diff --git a/_includes/v20.2/app/insecure/basic-sample.go b/_includes/v20.2/app/insecure/basic-sample.go new file mode 100644 index 00000000000..6a647f51641 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.go @@ -0,0 +1,44 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/lib/pq" +) + +func main() { + // Connect to the "bank" database. + db, err := sql.Open("postgres", "postgresql://maxroach@localhost:26257/bank?sslmode=disable") + if err != nil { + log.Fatal("error connecting to the database: ", err) + } + + // Create the "accounts" table. + if _, err := db.Exec( + "CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)"); err != nil { + log.Fatal(err) + } + + // Insert two rows into the "accounts" table. + if _, err := db.Exec( + "INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)"); err != nil { + log.Fatal(err) + } + + // Print out the balances. + rows, err := db.Query("SELECT id, balance FROM accounts") + if err != nil { + log.Fatal(err) + } + defer rows.Close() + fmt.Println("Initial balances:") + for rows.Next() { + var id, balance int + if err := rows.Scan(&id, &balance); err != nil { + log.Fatal(err) + } + fmt.Printf("%d %d\n", id, balance) + } +} diff --git a/_includes/v20.2/app/insecure/basic-sample.js b/_includes/v20.2/app/insecure/basic-sample.js new file mode 100644 index 00000000000..f89ea020a74 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.js @@ -0,0 +1,55 @@ +var async = require('async'); +var fs = require('fs'); +var pg = require('pg'); + +// Connect to the "bank" database. +var config = { + user: 'maxroach', + host: 'localhost', + database: 'bank', + port: 26257 +}; + +// Create a pool. +var pool = new pg.Pool(config); + +pool.connect(function (err, client, done) { + + // Close communication with the database and exit. + var finish = function () { + done(); + process.exit(); + }; + + if (err) { + console.error('could not connect to cockroachdb', err); + finish(); + } + async.waterfall([ + function (next) { + // Create the 'accounts' table. + client.query('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT);', next); + }, + function (results, next) { + // Insert two rows into the 'accounts' table. + client.query('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250);', next); + }, + function (results, next) { + // Print out account balances. + client.query('SELECT id, balance FROM accounts;', next); + }, + ], + function (err, results) { + if (err) { + console.error('Error inserting into and selecting from accounts: ', err); + finish(); + } + + console.log('Initial balances:'); + results.rows.forEach(function (row) { + console.log(row); + }); + + finish(); + }); +}); diff --git a/_includes/v20.2/app/insecure/basic-sample.php b/_includes/v20.2/app/insecure/basic-sample.php new file mode 100644 index 00000000000..db5a26e3111 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.php @@ -0,0 +1,20 @@ + PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_PERSISTENT => true + )); + + $dbh->exec('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)'); + + print "Account balances:\r\n"; + foreach ($dbh->query('SELECT id, balance FROM accounts') as $row) { + print $row['id'] . ': ' . $row['balance'] . "\r\n"; + } +} catch (Exception $e) { + print $e->getMessage() . "\r\n"; + exit(1); +} +?> diff --git a/_includes/v20.2/app/insecure/basic-sample.py b/_includes/v20.2/app/insecure/basic-sample.py new file mode 100644 index 00000000000..c0df43b8afc --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +import psycopg2 +import psycopg2.errorcodes +import time +import logging +import random + + +def create_accounts(conn): + with conn.cursor() as cur: + cur.execute('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)') + cur.execute('UPSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)') + logging.debug("create_accounts(): status message: {}".format(cur.statusmessage)) + conn.commit() + + +def print_balances(conn): + with conn.cursor() as cur: + cur.execute("SELECT id, balance FROM accounts") + logging.debug("print_balances(): status message: {}".format(cur.statusmessage)) + rows = cur.fetchall() + conn.commit() + print("Balances at {}".format(time.asctime())) + for row in rows: + print([str(cell) for cell in row]) + + +def delete_accounts(conn): + with conn.cursor() as cur: + cur.execute("DELETE FROM bank.accounts") + logging.debug("delete_accounts(): status message: {}".format(cur.statusmessage)) + conn.commit() + + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn, op): + retries = 0 + max_retries = 3 + with conn: + while True: + retries +=1 + if retries == max_retries: + err_msg = "Transaction did not succeed after {} retries".format(max_retries) + raise ValueError(err_msg) + + try: + op(conn) + + # If we reach this point, we were able to commit, so we break + # from the retry loop. + break + except psycopg2.Error as e: + logging.debug("e.pgcode: {}".format(e.pgcode)) + if e.pgcode == '40001': + # This is a retry error, so we roll back the current + # transaction and sleep for a bit before retrying. The + # sleep time increases for each failed transaction. + conn.rollback() + logging.debug("EXECUTE SERIALIZATION_FAILURE BRANCH") + sleep_ms = (2**retries) * 0.1 * (random.random() + 0.5) + logging.debug("Sleeping {} seconds".format(sleep_ms)) + time.sleep(sleep_ms) + continue + else: + logging.debug("EXECUTE NON-SERIALIZATION_FAILURE BRANCH") + raise e + + +# This function is used to test the transaction retry logic. It can be deleted +# from production code. +def test_retry_loop(conn): + with conn.cursor() as cur: + # The first statement in a transaction can be retried transparently on + # the server, so we need to add a dummy statement so that our + # force_retry() statement isn't the first one. + cur.execute('SELECT now()') + cur.execute("SELECT crdb_internal.force_retry('1s'::INTERVAL)") + logging.debug("test_retry_loop(): status message: {}".format(cur.statusmessage)) + + +def transfer_funds(conn, frm, to, amount): + with conn.cursor() as cur: + + # Check the current balance. + cur.execute("SELECT balance FROM accounts WHERE id = " + str(frm)) + from_balance = cur.fetchone()[0] + if from_balance < amount: + err_msg = "Insufficient funds in account {}: have {}, need {}".format(frm, from_balance, amount) + raise RuntimeError(err_msg) + + # Perform the transfer. + cur.execute("UPDATE accounts SET balance = balance - %s WHERE id = %s", + (amount, frm)) + cur.execute("UPDATE accounts SET balance = balance + %s WHERE id = %s", + (amount, to)) + conn.commit() + logging.debug("transfer_funds(): status message: {}".format(cur.statusmessage)) + + +def main(): + + dsn = 'postgresql://maxroach@localhost:26257/bank?sslmode=disable' + conn = psycopg2.connect(dsn) + + # Uncomment the below to turn on logging to the console. This was useful + # when testing transaction retry handling. It is not necessary for + # production code. + # log_level = getattr(logging, 'DEBUG', None) + # logging.basicConfig(level=log_level) + + create_accounts(conn) + + print_balances(conn) + + amount = 100 + fromId = 1 + toId = 2 + + try: + run_transaction(conn, lambda conn: transfer_funds(conn, fromId, toId, amount)) + + # The function below is used to test the transaction retry logic. It + # can be deleted from production code. + # run_transaction(conn, lambda conn: test_retry_loop(conn)) + except ValueError as ve: + # Below, we print the error and continue on so this example is easy to + # run (and run, and run...). In real code you should handle this error + # and any others thrown by the database interaction. + logging.debug("run_transaction(conn, op) failed: {}".format(ve)) + pass + + print_balances(conn) + + delete_accounts(conn) + + # Close communication with the database. + conn.close() + + +if __name__ == '__main__': + main() diff --git a/_includes/v20.2/app/insecure/basic-sample.rb b/_includes/v20.2/app/insecure/basic-sample.rb new file mode 100644 index 00000000000..904460381f6 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.rb @@ -0,0 +1,28 @@ +# Import the driver. +require 'pg' + +# Connect to the "bank" database. +conn = PG.connect( + user: 'maxroach', + dbname: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'disable' +) + +# Create the "accounts" table. +conn.exec('CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)') + +# Insert two rows into the "accounts" table. +conn.exec('INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)') + +# Print out the balances. +puts 'Initial balances:' +conn.exec('SELECT id, balance FROM accounts') do |res| + res.each do |row| + puts row + end +end + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/insecure/basic-sample.rs b/_includes/v20.2/app/insecure/basic-sample.rs new file mode 100644 index 00000000000..8b7c3b115a9 --- /dev/null +++ b/_includes/v20.2/app/insecure/basic-sample.rs @@ -0,0 +1,32 @@ +use postgres::{Client, NoTls}; + +fn main() { + let mut client = Client::connect("postgresql://maxroach@localhost:26257/bank", NoTls).unwrap(); + + // Create the "accounts" table. + client + .execute( + "CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", + &[], + ) + .unwrap(); + + // Insert two rows into the "accounts" table. + client + .execute( + "INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250)", + &[], + ) + .unwrap(); + + // Print out the balances. + println!("Initial balances:"); + for row in &client + .query("SELECT id, balance FROM accounts", &[]) + .unwrap() + { + let id: i64 = row.get(0); + let balance: i64 = row.get(1); + println!("{} {}", id, balance); + } +} diff --git a/_includes/v20.2/app/insecure/create-maxroach-user-and-bank-database.md b/_includes/v20.2/app/insecure/create-maxroach-user-and-bank-database.md new file mode 100644 index 00000000000..5beb4cdd508 --- /dev/null +++ b/_includes/v20.2/app/insecure/create-maxroach-user-and-bank-database.md @@ -0,0 +1,32 @@ +Start the [built-in SQL shell](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +In the SQL shell, issue the following statements to create the `maxroach` user and `bank` database: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER IF NOT EXISTS maxroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +Give the `maxroach` user the necessary permissions: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE bank TO maxroach; +~~~ + +Exit the SQL shell: + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ diff --git a/_includes/v20.2/app/insecure/django-basic-sample/models.py b/_includes/v20.2/app/insecure/django-basic-sample/models.py new file mode 100644 index 00000000000..6068f8bbb8e --- /dev/null +++ b/_includes/v20.2/app/insecure/django-basic-sample/models.py @@ -0,0 +1,17 @@ +from django.db import models + +class Customers(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=250) + +class Products(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=250) + price = models.DecimalField(max_digits=18, decimal_places=2) + +class Orders(models.Model): + id = models.AutoField(primary_key=True) + subtotal = models.DecimalField(max_digits=18, decimal_places=2) + customer = models.ForeignKey(Customers, on_delete=models.CASCADE, null=True) + product = models.ManyToManyField(Products) + diff --git a/_includes/v20.2/app/insecure/django-basic-sample/settings.py b/_includes/v20.2/app/insecure/django-basic-sample/settings.py new file mode 100644 index 00000000000..d23c128e33f --- /dev/null +++ b/_includes/v20.2/app/insecure/django-basic-sample/settings.py @@ -0,0 +1,124 @@ +""" +Django settings for myproject project. + +Generated by 'django-admin startproject' using Django 3.0. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'spl=g73)8-)ja%x*k1eje4d#&24#t)zao^s$6vc1rdk(e3t!e(' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = ['0.0.0.0'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'myproject', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'myproject.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'myproject.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django_cockroachdb', + 'NAME': 'bank', + 'USER': 'django', + 'HOST': 'localhost', + 'PORT': '26257', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/_includes/v20.2/app/insecure/django-basic-sample/urls.py b/_includes/v20.2/app/insecure/django-basic-sample/urls.py new file mode 100644 index 00000000000..9550d713ffa --- /dev/null +++ b/_includes/v20.2/app/insecure/django-basic-sample/urls.py @@ -0,0 +1,20 @@ +from django.contrib import admin +from django.urls import path + +from .views import CustomersView, OrdersView, PingView, ProductView + +urlpatterns = [ + path('admin/', admin.site.urls), + + path('ping/', PingView.as_view()), + + # Endpoints for customers URL. + path('customer/', CustomersView.as_view(), name='customers'), + path('customer//', CustomersView.as_view(), name='customers'), + + # Endpoints for customers URL. + path('product/', ProductView.as_view(), name='product'), + path('product//', ProductView.as_view(), name='product'), + + path('order/', OrdersView.as_view(), name='order'), +] diff --git a/_includes/v20.2/app/insecure/django-basic-sample/views.py b/_includes/v20.2/app/insecure/django-basic-sample/views.py new file mode 100644 index 00000000000..78143916ee8 --- /dev/null +++ b/_includes/v20.2/app/insecure/django-basic-sample/views.py @@ -0,0 +1,107 @@ +from django.http import JsonResponse, HttpResponse +from django.utils.decorators import method_decorator +from django.views.generic import View +from django.views.decorators.csrf import csrf_exempt +from django.db import Error, IntegrityError +from django.db.transaction import atomic + +import json +import sys +import time + +from .models import * + +# Warning: Do not use retry_on_exception in an inner nested transaction. +def retry_on_exception(num_retries=3, on_failure=HttpResponse(status=500), delay_=0.5, backoff_=1.5): + def retry(view): + def wrapper(*args, **kwargs): + delay = delay_ + for i in range(num_retries): + try: + return view(*args, **kwargs) + except IntegrityError as ex: + if i == num_retries - 1: + return on_failure + elif getattr(ex.__cause__, 'pgcode', '') == errorcodes.SERIALIZATION_FAILURE: + time.sleep(delay) + delay *= backoff_ + except Error as ex: + return on_failure + return wrapper + return retry + +class PingView(View): + def get(self, request, *args, **kwargs): + return HttpResponse("python/django", status=200) + +@method_decorator(csrf_exempt, name='dispatch') +class CustomersView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + customers = list(Customers.objects.values()) + else: + customers = list(Customers.objects.filter(id=id).values()) + return JsonResponse(customers, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + name = form_data['name'] + c = Customers(name=name) + c.save() + return HttpResponse(status=200) + + @retry_on_exception(3) + @atomic + def delete(self, request, id=None, *args, **kwargs): + if id is None: + return HttpResponse(status=404) + Customers.objects.filter(id=id).delete() + return HttpResponse(status=200) + + # The PUT method is shadowed by the POST method, so there doesn't seem + # to be a reason to include it. + +@method_decorator(csrf_exempt, name='dispatch') +class ProductView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + products = list(Products.objects.values()) + else: + products = list(Products.objects.filter(id=id).values()) + return JsonResponse(products, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + name, price = form_data['name'], form_data['price'] + p = Products(name=name, price=price) + p.save() + return HttpResponse(status=200) + + # The REST API outlined in the github does not say that /product/ needs + # a PUT and DELETE method + +@method_decorator(csrf_exempt, name='dispatch') +class OrdersView(View): + def get(self, request, id=None, *args, **kwargs): + if id is None: + orders = list(Orders.objects.values()) + else: + orders = list(Orders.objects.filter(id=id).values()) + return JsonResponse(orders, safe=False) + + @retry_on_exception(3) + @atomic + def post(self, request, *args, **kwargs): + form_data = json.loads(request.body.decode()) + c = Customers.objects.get(id=form_data['customer']['id']) + o = Orders(subtotal=form_data['subtotal'], customer=c) + o.save() + for p in form_data['products']: + p = Products.objects.get(id=p['id']) + o.product.add(p) + o.save() + return HttpResponse(status=200) diff --git a/_includes/v20.2/app/insecure/gorm-basic-sample.go b/_includes/v20.2/app/insecure/gorm-basic-sample.go new file mode 100644 index 00000000000..b8529962c2b --- /dev/null +++ b/_includes/v20.2/app/insecure/gorm-basic-sample.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "log" + + // Import GORM-related packages. + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" +) + +// Account is our model, which corresponds to the "accounts" database table. +type Account struct { + ID int `gorm:"primary_key"` + Balance int +} + +func main() { + // Connect to the "bank" database as the "maxroach" user. + const addr = "postgresql://maxroach@localhost:26257/bank?sslmode=disable" + db, err := gorm.Open("postgres", addr) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Automatically create the "accounts" table based on the Account model. + db.AutoMigrate(&Account{}) + + // Insert two rows into the "accounts" table. + db.Create(&Account{ID: 1, Balance: 1000}) + db.Create(&Account{ID: 2, Balance: 250}) + + // Print out the balances. + var accounts []Account + db.Find(&accounts) + fmt.Println("Initial balances:") + for _, account := range accounts { + fmt.Printf("%d %d\n", account.ID, account.Balance) + } +} diff --git a/_includes/v20.2/app/insecure/gorm-sample.go b/_includes/v20.2/app/insecure/gorm-sample.go new file mode 100644 index 00000000000..cf12ee40ebc --- /dev/null +++ b/_includes/v20.2/app/insecure/gorm-sample.go @@ -0,0 +1,206 @@ +package main + +import ( + "fmt" + "log" + "math" + "math/rand" + "time" + + // Import GORM-related packages. + "github.com/jinzhu/gorm" + _ "github.com/jinzhu/gorm/dialects/postgres" + + // Necessary in order to check for transaction retry error codes. + "github.com/lib/pq" +) + +// Account is our model, which corresponds to the "accounts" database +// table. +type Account struct { + ID int `gorm:"primary_key"` + Balance int +} + +// Functions of type `txnFunc` are passed as arguments to our +// `runTransaction` wrapper that handles transaction retries for us +// (see implementation below). +type txnFunc func(*gorm.DB) error + +// This function is used for testing the transaction retry loop. It +// can be deleted from production code. +var forceRetryLoop txnFunc = func(db *gorm.DB) error { + + // The first statement in a transaction can be retried transparently + // on the server, so we need to add a dummy statement so that our + // force_retry statement isn't the first one. + if err := db.Exec("SELECT now()").Error; err != nil { + return err + } + // Used to force a transaction retry. + if err := db.Exec("SELECT crdb_internal.force_retry('1s'::INTERVAL)").Error; err != nil { + return err + } + return nil +} + +func transferFunds(db *gorm.DB, fromID int, toID int, amount int) error { + var fromAccount Account + var toAccount Account + + db.First(&fromAccount, fromID) + db.First(&toAccount, toID) + + if fromAccount.Balance < amount { + return fmt.Errorf("account %d balance %d is lower than transfer amount %d", fromAccount.ID, fromAccount.Balance, amount) + } + + fromAccount.Balance -= amount + toAccount.Balance += amount + + if err := db.Save(&fromAccount).Error; err != nil { + return err + } + if err := db.Save(&toAccount).Error; err != nil { + return err + } + return nil +} + +func main() { + // Connect to the "bank" database as the "maxroach" user. + const addr = "postgresql://maxroach@localhost:26257/bank?sslmode=disable" + db, err := gorm.Open("postgres", addr) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + // Set to `true` and GORM will print out all DB queries. + db.LogMode(false) + + // Automatically create the "accounts" table based on the Account + // model. + db.AutoMigrate(&Account{}) + + // Insert two rows into the "accounts" table. + var fromID = 1 + var toID = 2 + db.Create(&Account{ID: fromID, Balance: 1000}) + db.Create(&Account{ID: toID, Balance: 250}) + + // The sequence of steps in this section is: + // 1. Print account balances. + // 2. Set up some Accounts and transfer funds between them inside + // a transaction. + // 3. Print account balances again to verify the transfer occurred. + + // Print balances before transfer. + printBalances(db) + + // The amount to be transferred between the accounts. + var amount = 100 + + // Transfer funds between accounts. To handle potential + // transaction retry errors, we wrap the call to `transferFunds` + // in `runTransaction`, a wrapper which implements a retry loop + // with exponential backoff around our access to the database (see + // the implementation for details). + if err := runTransaction(db, + func(*gorm.DB) error { + return transferFunds(db, fromID, toID, amount) + }, + ); err != nil { + // If the error is returned, it's either: + // 1. Not a transaction retry error, i.e., some other kind + // of database error that you should handle here. + // 2. A transaction retry error that has occurred more than + // N times (defined by the `maxRetries` variable inside + // `runTransaction`), in which case you will need to figure + // out why your database access is resulting in so much + // contention (see 'Understanding and avoiding transaction + // contention': + // https://www.cockroachlabs.com/docs/stable/performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) + fmt.Println(err) + } + + // Print balances after transfer to ensure that it worked. + printBalances(db) + + // Delete accounts so we can start fresh when we want to run this + // program again. + deleteAccounts(db) +} + +// Wrapper for a transaction. This automatically re-calls `fn` with +// the open transaction as an argument as long as the database server +// asks for the transaction to be retried. +func runTransaction(db *gorm.DB, fn txnFunc) error { + var maxRetries = 3 + for retries := 0; retries <= maxRetries; retries++ { + if retries == maxRetries { + return fmt.Errorf("hit max of %d retries, aborting", retries) + } + txn := db.Begin() + if err := fn(txn); err != nil { + // We need to cast GORM's db.Error to *pq.Error so we can + // detect the Postgres transaction retry error code and + // handle retries appropriately. + pqErr := err.(*pq.Error) + if pqErr.Code == "40001" { + // Since this is a transaction retry error, we + // ROLLBACK the transaction and sleep a little before + // trying again. Each time through the loop we sleep + // for a little longer than the last time + // (A.K.A. exponential backoff). + txn.Rollback() + var sleepMs = math.Pow(2, float64(retries)) * 100 * (rand.Float64() + 0.5) + fmt.Printf("Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMs) + time.Sleep(time.Millisecond * time.Duration(sleepMs)) + } else { + // If it's not a retry error, it's some other sort of + // DB interaction error that needs to be handled by + // the caller. + return err + } + } else { + // All went well, so we try to commit and break out of the + // retry loop if possible. + if err := txn.Commit().Error; err != nil { + pqErr := err.(*pq.Error) + if pqErr.Code == "40001" { + // However, our attempt to COMMIT could also + // result in a retry error, in which case we + // continue back through the loop and try again. + continue + } else { + // If it's not a retry error, it's some other sort + // of DB interaction error that needs to be + // handled by the caller. + return err + } + } + break + } + } + return nil +} + +func printBalances(db *gorm.DB) { + var accounts []Account + db.Find(&accounts) + fmt.Printf("Balance at '%s':\n", time.Now()) + for _, account := range accounts { + fmt.Printf("%d %d\n", account.ID, account.Balance) + } +} + +func deleteAccounts(db *gorm.DB) error { + // Used to tear down the accounts table so we can re-run this + // program. + err := db.Exec("DELETE from accounts where ID > 0").Error + if err != nil { + return err + } + return nil +} diff --git a/_includes/v20.2/app/insecure/hibernate-basic-sample/Sample.java b/_includes/v20.2/app/insecure/hibernate-basic-sample/Sample.java new file mode 100644 index 00000000000..58d28f37a4b --- /dev/null +++ b/_includes/v20.2/app/insecure/hibernate-basic-sample/Sample.java @@ -0,0 +1,236 @@ +package com.cockroachlabs; + +import org.hibernate.Session; +import org.hibernate.SessionFactory; +import org.hibernate.Transaction; +import org.hibernate.JDBCException; +import org.hibernate.cfg.Configuration; + +import java.util.*; +import java.util.function.Function; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +public class Sample { + + private static final Random RAND = new Random(); + private static final boolean FORCE_RETRY = false; + private static final String RETRY_SQL_STATE = "40001"; + private static final int MAX_ATTEMPT_COUNT = 6; + + // Account is our model, which corresponds to the "accounts" database table. + @Entity + @Table(name="accounts") + public static class Account { + @Id + @Column(name="id") + public long id; + + public long getId() { + return id; + } + + @Column(name="balance") + public long balance; + public long getBalance() { + return balance; + } + public void setBalance(long newBalance) { + this.balance = newBalance; + } + + // Convenience constructor. + public Account(int id, int balance) { + this.id = id; + this.balance = balance; + } + + // Hibernate needs a default (no-arg) constructor to create model objects. + public Account() {} + } + + private static Function addAccounts() throws JDBCException{ + Function f = s -> { + long rv = 0; + try { + s.save(new Account(1, 1000)); + s.save(new Account(2, 250)); + s.save(new Account(3, 314159)); + rv = 1; + System.out.printf("APP: addAccounts() --> %d\n", rv); + } catch (JDBCException e) { + throw e; + } + return rv; + }; + return f; + } + + private static Function transferFunds(long fromId, long toId, long amount) throws JDBCException{ + Function f = s -> { + long rv = 0; + try { + Account fromAccount = (Account) s.get(Account.class, fromId); + Account toAccount = (Account) s.get(Account.class, toId); + if (!(amount > fromAccount.getBalance())) { + fromAccount.balance -= amount; + toAccount.balance += amount; + s.save(fromAccount); + s.save(toAccount); + rv = amount; + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d\n", fromId, toId, amount, rv); + } + } catch (JDBCException e) { + throw e; + } + return rv; + }; + return f; + } + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // intended for production code. + private static Function forceRetryLogic() throws JDBCException { + Function f = s -> { + long rv = -1; + try { + System.out.printf("APP: testRetryLogic: BEFORE EXCEPTION\n"); + s.createNativeQuery("SELECT crdb_internal.force_retry('1s')").executeUpdate(); + } catch (JDBCException e) { + System.out.printf("APP: testRetryLogic: AFTER EXCEPTION\n"); + throw e; + } + return rv; + }; + return f; + } + + private static Function getAccountBalance(long id) throws JDBCException{ + Function f = s -> { + long balance; + try { + Account account = s.get(Account.class, id); + balance = account.getBalance(); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", id, balance); + } catch (JDBCException e) { + throw e; + } + return balance; + }; + return f; + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic so we don't have to duplicate it in + // various places. + private static long runTransaction(Session session, Function fn) { + long rv = 0; + int attemptCount = 0; + + while (attemptCount < MAX_ATTEMPT_COUNT) { + attemptCount++; + + if (attemptCount > 1) { + System.out.printf("APP: Entering retry loop again, iteration %d\n", attemptCount); + } + + Transaction txn = session.beginTransaction(); + System.out.printf("APP: BEGIN;\n"); + + if (attemptCount == MAX_ATTEMPT_COUNT) { + String err = String.format("hit max of %s attempts, aborting", MAX_ATTEMPT_COUNT); + throw new RuntimeException(err); + } + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryLogic()'. + if (FORCE_RETRY) { + session.createNativeQuery("SELECT now()").list(); + } + + try { + rv = fn.apply(session); + if (rv != -1) { + txn.commit(); + System.out.printf("APP: COMMIT;\n"); + break; + } + } catch (JDBCException e) { + if (RETRY_SQL_STATE.equals(e.getSQLState())) { + // Since this is a transaction retry error, we + // roll back the transaction and sleep a little + // before trying again. Each time through the + // loop we sleep for a little longer than the last + // time (A.K.A. exponential backoff). + System.out.printf("APP: retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", e.getSQLState(), e.getMessage(), attemptCount); + System.out.printf("APP: ROLLBACK;\n"); + txn.rollback(); + int sleepMillis = (int)(Math.pow(2, attemptCount) * 100) + RAND.nextInt(100); + System.out.printf("APP: Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMillis); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException ignored) { + // no-op + } + rv = -1; + } else { + throw e; + } + } + } + return rv; + } + + public static void main(String[] args) { + // Create a SessionFactory based on our hibernate.cfg.xml configuration + // file, which defines how to connect to the database. + SessionFactory sessionFactory = + new Configuration() + .configure("hibernate.cfg.xml") + .addAnnotatedClass(Account.class) + .buildSessionFactory(); + + try (Session session = sessionFactory.openSession()) { + long fromAccountId = 1; + long toAccountId = 2; + long transferAmount = 100; + + if (FORCE_RETRY) { + System.out.printf("APP: About to test retry logic in 'runTransaction'\n"); + runTransaction(session, forceRetryLogic()); + } else { + + runTransaction(session, addAccounts()); + long fromBalance = runTransaction(session, getAccountBalance(fromAccountId)); + long toBalance = runTransaction(session, getAccountBalance(toAccountId)); + if (fromBalance != -1 && toBalance != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalance); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalance); + } + + // Transfer $100 from account 1 to account 2 + long transferResult = runTransaction(session, transferFunds(fromAccountId, toAccountId, transferAmount)); + if (transferResult != -1) { + // Success! + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d \n", fromAccountId, toAccountId, transferAmount, transferResult); + + long fromBalanceAfter = runTransaction(session, getAccountBalance(fromAccountId)); + long toBalanceAfter = runTransaction(session, getAccountBalance(toAccountId)); + if (fromBalanceAfter != -1 && toBalanceAfter != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalanceAfter); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalanceAfter); + } + } + } + } finally { + sessionFactory.close(); + } + } +} diff --git a/_includes/v20.2/app/insecure/hibernate-basic-sample/build.gradle b/_includes/v20.2/app/insecure/hibernate-basic-sample/build.gradle new file mode 100644 index 00000000000..36f33d73fe6 --- /dev/null +++ b/_includes/v20.2/app/insecure/hibernate-basic-sample/build.gradle @@ -0,0 +1,16 @@ +group 'com.cockroachlabs' +version '1.0' + +apply plugin: 'java' +apply plugin: 'application' + +mainClassName = 'com.cockroachlabs.Sample' + +repositories { + mavenCentral() +} + +dependencies { + compile 'org.hibernate:hibernate-core:5.2.4.Final' + compile 'org.postgresql:postgresql:42.2.2.jre7' +} diff --git a/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate-basic-sample.tgz b/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate-basic-sample.tgz new file mode 100644 index 00000000000..8205b379229 Binary files /dev/null and b/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate-basic-sample.tgz differ diff --git a/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate.cfg.xml b/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate.cfg.xml new file mode 100644 index 00000000000..ad27c7d746c --- /dev/null +++ b/_includes/v20.2/app/insecure/hibernate-basic-sample/hibernate.cfg.xml @@ -0,0 +1,20 @@ + + + + + + org.postgresql.Driver + org.hibernate.dialect.PostgreSQL95Dialect + jdbc:postgresql://127.0.0.1:26257/bank?sslmode=disable + maxroach + + + create + + + true + true + + diff --git a/_includes/v20.2/app/insecure/jooq-basic-sample/Sample.java b/_includes/v20.2/app/insecure/jooq-basic-sample/Sample.java new file mode 100644 index 00000000000..fdb8aa3115d --- /dev/null +++ b/_includes/v20.2/app/insecure/jooq-basic-sample/Sample.java @@ -0,0 +1,215 @@ +package com.cockroachlabs; + +import com.cockroachlabs.example.jooq.db.Tables; +import com.cockroachlabs.example.jooq.db.tables.records.AccountsRecord; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.Source; +import org.jooq.conf.RenderQuotedNames; +import org.jooq.conf.Settings; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +import static com.cockroachlabs.example.jooq.db.Tables.ACCOUNTS; + +public class Sample { + + private static final Random RAND = new Random(); + private static final boolean FORCE_RETRY = false; + private static final String RETRY_SQL_STATE = "40001"; + private static final int MAX_ATTEMPT_COUNT = 6; + + private static Function addAccounts() { + return ctx -> { + long rv = 0; + + ctx.delete(ACCOUNTS).execute(); + ctx.batchInsert( + new AccountsRecord(1L, 1000L), + new AccountsRecord(2L, 250L), + new AccountsRecord(3L, 314159L) + ).execute(); + + rv = 1; + System.out.printf("APP: addAccounts() --> %d\n", rv); + return rv; + }; + } + + private static Function transferFunds(long fromId, long toId, long amount) { + return ctx -> { + long rv = 0; + + AccountsRecord fromAccount = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(fromId)); + AccountsRecord toAccount = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(toId)); + + if (!(amount > fromAccount.getBalance())) { + fromAccount.setBalance(fromAccount.getBalance() - amount); + toAccount.setBalance(toAccount.getBalance() + amount); + + ctx.batchUpdate(fromAccount, toAccount).execute(); + rv = amount; + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d\n", fromId, toId, amount, rv); + } + + return rv; + }; + } + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // intended for production code. + private static Function forceRetryLogic() { + return ctx -> { + long rv = -1; + try { + System.out.printf("APP: testRetryLogic: BEFORE EXCEPTION\n"); + ctx.execute("SELECT crdb_internal.force_retry('1s')"); + } catch (DataAccessException e) { + System.out.printf("APP: testRetryLogic: AFTER EXCEPTION\n"); + throw e; + } + return rv; + }; + } + + private static Function getAccountBalance(long id) { + return ctx -> { + AccountsRecord account = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(id)); + long balance = account.getBalance(); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", id, balance); + return balance; + }; + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic so we don't have to duplicate it in + // various places. + private static long runTransaction(DSLContext session, Function fn) { + AtomicLong rv = new AtomicLong(0L); + AtomicInteger attemptCount = new AtomicInteger(0); + + while (attemptCount.get() < MAX_ATTEMPT_COUNT) { + attemptCount.incrementAndGet(); + + if (attemptCount.get() > 1) { + System.out.printf("APP: Entering retry loop again, iteration %d\n", attemptCount.get()); + } + + if (session.connectionResult(connection -> { + connection.setAutoCommit(false); + System.out.printf("APP: BEGIN;\n"); + + if (attemptCount.get() == MAX_ATTEMPT_COUNT) { + String err = String.format("hit max of %s attempts, aborting", MAX_ATTEMPT_COUNT); + throw new RuntimeException(err); + } + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryLogic()'. + if (FORCE_RETRY) { + session.fetch("SELECT now()"); + } + + try { + rv.set(fn.apply(session)); + if (rv.get() != -1) { + connection.commit(); + System.out.printf("APP: COMMIT;\n"); + return true; + } + } catch (DataAccessException | SQLException e) { + String sqlState = e instanceof SQLException ? ((SQLException) e).getSQLState() : ((DataAccessException) e).sqlState(); + + if (RETRY_SQL_STATE.equals(sqlState)) { + // Since this is a transaction retry error, we + // roll back the transaction and sleep a little + // before trying again. Each time through the + // loop we sleep for a little longer than the last + // time (A.K.A. exponential backoff). + System.out.printf("APP: retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", sqlState, e.getMessage(), attemptCount.get()); + System.out.printf("APP: ROLLBACK;\n"); + connection.rollback(); + int sleepMillis = (int)(Math.pow(2, attemptCount.get()) * 100) + RAND.nextInt(100); + System.out.printf("APP: Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMillis); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException ignored) { + // no-op + } + rv.set(-1L); + } else { + throw e; + } + } + + return false; + })) { + break; + } + } + + return rv.get(); + } + + public static void main(String[] args) throws Exception { + try (Connection connection = DriverManager.getConnection( + "jdbc:postgresql://localhost:26257/bank?sslmode=disable", + "maxroach", + "" + )) { + DSLContext ctx = DSL.using(connection, SQLDialect.COCKROACHDB, new Settings() + .withExecuteLogging(true) + .withRenderQuotedNames(RenderQuotedNames.NEVER)); + + // Initialise database with db.sql script + try (InputStream in = Sample.class.getResourceAsStream("/db.sql")) { + ctx.parser().parse(Source.of(in).readString()).executeBatch(); + } + + long fromAccountId = 1; + long toAccountId = 2; + long transferAmount = 100; + + if (FORCE_RETRY) { + System.out.printf("APP: About to test retry logic in 'runTransaction'\n"); + runTransaction(ctx, forceRetryLogic()); + } else { + + runTransaction(ctx, addAccounts()); + long fromBalance = runTransaction(ctx, getAccountBalance(fromAccountId)); + long toBalance = runTransaction(ctx, getAccountBalance(toAccountId)); + if (fromBalance != -1 && toBalance != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalance); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalance); + } + + // Transfer $100 from account 1 to account 2 + long transferResult = runTransaction(ctx, transferFunds(fromAccountId, toAccountId, transferAmount)); + if (transferResult != -1) { + // Success! + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d \n", fromAccountId, toAccountId, transferAmount, transferResult); + + long fromBalanceAfter = runTransaction(ctx, getAccountBalance(fromAccountId)); + long toBalanceAfter = runTransaction(ctx, getAccountBalance(toAccountId)); + if (fromBalanceAfter != -1 && toBalanceAfter != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalanceAfter); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalanceAfter); + } + } + } + } + } +} diff --git a/_includes/v20.2/app/insecure/jooq-basic-sample/jooq-basic-sample.zip b/_includes/v20.2/app/insecure/jooq-basic-sample/jooq-basic-sample.zip new file mode 100644 index 00000000000..f11f86b8f43 Binary files /dev/null and b/_includes/v20.2/app/insecure/jooq-basic-sample/jooq-basic-sample.zip differ diff --git a/_includes/v20.2/app/insecure/pony-basic-sample.py b/_includes/v20.2/app/insecure/pony-basic-sample.py new file mode 100644 index 00000000000..3d367179d5b --- /dev/null +++ b/_includes/v20.2/app/insecure/pony-basic-sample.py @@ -0,0 +1,69 @@ +import random +from math import floor +from pony.orm import * + +db = Database() + +# The Account class corresponds to the "accounts" database table. + + +class Account(db.Entity): + _table_ = 'accounts' + id = PrimaryKey(int) + balance = Required(int) + + +db_params = dict(provider='cockroach', user='maxroach', + host='localhost', port=26257, database='bank', sslmode='disable') + + +sql_debug(True) # Print all generated SQL queries to stdout +db.bind(**db_params) # Bind Database object to the real database +db.generate_mapping(create_tables=True) # Create tables + + +# Store the account IDs we create for later use. + +seen_account_ids = set() + + +# The code below generates random IDs for new accounts. + +@db_session # db_session decorator manages the transactions +def create_random_accounts(n): + elems = iter(range(n)) + for i in elems: + billion = 1000000000 + new_id = floor(random.random() * billion) + seen_account_ids.add(new_id) + # Create new account + Account(id=new_id, balance=floor(random.random() * 1000000)) + + +create_random_accounts(100) + + +def get_random_account_id(): + id = random.choice(tuple(seen_account_ids)) + return id + + +@db_session(retry=10) # retry of the optimistic transaction +def transfer_funds_randomly(): + """ + Cuts a randomly selected account's balance in half, and gives the + other half to some other randomly selected account. + """ + + source_id = get_random_account_id() + sink_id = get_random_account_id() + + source = Account.get(id=source_id) + amount = floor(source.balance / 2) + + if source.balance < amount: + raise "Insufficient funds" + + source.balance -= amount + sink = Account.get(id=sink_id) + sink.balance += amount diff --git a/_includes/v20.2/app/insecure/sequelize-basic-sample.js b/_includes/v20.2/app/insecure/sequelize-basic-sample.js new file mode 100644 index 00000000000..ca92b98e375 --- /dev/null +++ b/_includes/v20.2/app/insecure/sequelize-basic-sample.js @@ -0,0 +1,35 @@ +var Sequelize = require('sequelize-cockroachdb'); + +// Connect to CockroachDB through Sequelize. +var sequelize = new Sequelize('bank', 'maxroach', '', { + dialect: 'postgres', + port: 26257, + logging: false +}); + +// Define the Account model for the "accounts" table. +var Account = sequelize.define('accounts', { + id: { type: Sequelize.INTEGER, primaryKey: true }, + balance: { type: Sequelize.INTEGER } +}); + +// Create the "accounts" table. +Account.sync({force: true}).then(function() { + // Insert two rows into the "accounts" table. + return Account.bulkCreate([ + {id: 1, balance: 1000}, + {id: 2, balance: 250} + ]); +}).then(function() { + // Retrieve accounts. + return Account.findAll(); +}).then(function(accounts) { + // Print out the balances. + accounts.forEach(function(account) { + console.log(account.id + ' ' + account.balance); + }); + process.exit(0); +}).catch(function(err) { + console.error('error: ' + err.message); + process.exit(1); +}); diff --git a/_includes/v20.2/app/insecure/txn-sample.clj b/_includes/v20.2/app/insecure/txn-sample.clj new file mode 100644 index 00000000000..0e2d9df55e3 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.clj @@ -0,0 +1,44 @@ +(ns test.test + (:require [clojure.java.jdbc :as j] + [test.util :as util])) + +;; Define the connection parameters to the cluster. +(def db-spec {:dbtype "postgresql" + :dbname "bank" + :host "localhost" + :port "26257" + :user "maxroach"}) + +;; The transaction we want to run. +(defn transferFunds + [txn from to amount] + + ;; Check the current balance. + (let [fromBalance (->> (j/query txn ["SELECT balance FROM accounts WHERE id = ?" from]) + (mapv :balance) + (first))] + (when (< fromBalance amount) + (throw (Exception. "Insufficient funds")))) + + ;; Perform the transfer. + (j/execute! txn [(str "UPDATE accounts SET balance = balance - " amount " WHERE id = " from)]) + (j/execute! txn [(str "UPDATE accounts SET balance = balance + " amount " WHERE id = " to)])) + +(defn test-txn [] + ;; Connect to the cluster and run the code below with + ;; the connection object bound to 'conn'. + (j/with-db-connection [conn db-spec] + + ;; Execute the transaction within an automatic retry block; + ;; the transaction object is bound to 'txn'. + (util/with-txn-retry [txn conn] + (transferFunds txn 1 2 100)) + + ;; Execute a query outside of an automatic retry block. + (println "Balances after transfer:") + (->> (j/query conn ["SELECT id, balance FROM accounts"]) + (map println) + (doall)))) + +(defn -main [& args] + (test-txn)) diff --git a/_includes/v20.2/app/insecure/txn-sample.cpp b/_includes/v20.2/app/insecure/txn-sample.cpp new file mode 100644 index 00000000000..0f65137be22 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.cpp @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; + +void transferFunds( + pqxx::dbtransaction *tx, int from, int to, int amount) { + // Read the balance. + pqxx::result r = tx->exec( + "SELECT balance FROM accounts WHERE id = " + to_string(from)); + assert(r.size() == 1); + int fromBalance = r[0][0].as(); + + if (fromBalance < amount) { + throw domain_error("insufficient funds"); + } + + // Perform the transfer. + tx->exec("UPDATE accounts SET balance = balance - " + + to_string(amount) + " WHERE id = " + to_string(from)); + tx->exec("UPDATE accounts SET balance = balance + " + + to_string(amount) + " WHERE id = " + to_string(to)); +} + + +// ExecuteTx runs fn inside a transaction and retries it as needed. +// On non-retryable failures, the transaction is aborted and rolled +// back; on success, the transaction is committed. +// +// For more information about CockroachDB's transaction model see +// https://cockroachlabs.com/docs/transactions.html. +// +// NOTE: the supplied exec closure should not have external side +// effects beyond changes to the database. +void executeTx( + pqxx::connection *c, function fn) { + pqxx::work tx(*c); + while (true) { + try { + pqxx::subtransaction s(tx, "cockroach_restart"); + fn(&s); + s.commit(); + break; + } catch (const pqxx::pqxx_exception& e) { + // Swallow "transaction restart" errors; the transaction will be retried. + // Unfortunately libpqxx doesn't give us access to the error code, so we + // do string matching to identify retryable errors. + if (string(e.base().what()).find("restart transaction:") == string::npos) { + throw; + } + } + } + tx.commit(); +} + +int main() { + try { + pqxx::connection c("postgresql://maxroach@localhost:26257/bank"); + + executeTx(&c, [](pqxx::dbtransaction *tx) { + transferFunds(tx, 1, 2, 100); + }); + } + catch (const exception &e) { + cerr << e.what() << endl; + return 1; + } + cout << "Success" << endl; + return 0; +} diff --git a/_includes/v20.2/app/insecure/txn-sample.cs b/_includes/v20.2/app/insecure/txn-sample.cs new file mode 100644 index 00000000000..f64a664ccff --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.cs @@ -0,0 +1,120 @@ +using System; +using System.Data; +using Npgsql; + +namespace Cockroach +{ + class MainClass + { + static void Main(string[] args) + { + var connStringBuilder = new NpgsqlConnectionStringBuilder(); + connStringBuilder.Host = "localhost"; + connStringBuilder.Port = 26257; + connStringBuilder.SslMode = SslMode.Disable; + connStringBuilder.Username = "maxroach"; + connStringBuilder.Database = "bank"; + TxnSample(connStringBuilder.ConnectionString); + } + + static void TransferFunds(NpgsqlConnection conn, NpgsqlTransaction tran, int from, int to, int amount) + { + int balance = 0; + using (var cmd = new NpgsqlCommand(String.Format("SELECT balance FROM accounts WHERE id = {0}", from), conn, tran)) + using (var reader = cmd.ExecuteReader()) + { + if (reader.Read()) + { + balance = reader.GetInt32(0); + } + else + { + throw new DataException(String.Format("Account id={0} not found", from)); + } + } + if (balance < amount) + { + throw new DataException(String.Format("Insufficient balance in account id={0}", from)); + } + using (var cmd = new NpgsqlCommand(String.Format("UPDATE accounts SET balance = balance - {0} where id = {1}", amount, from), conn, tran)) + { + cmd.ExecuteNonQuery(); + } + using (var cmd = new NpgsqlCommand(String.Format("UPDATE accounts SET balance = balance + {0} where id = {1}", amount, to), conn, tran)) + { + cmd.ExecuteNonQuery(); + } + } + + static void TxnSample(string connString) + { + using (var conn = new NpgsqlConnection(connString)) + { + conn.Open(); + + // Create the "accounts" table. + new NpgsqlCommand("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", conn).ExecuteNonQuery(); + + // Insert two rows into the "accounts" table. + using (var cmd = new NpgsqlCommand()) + { + cmd.Connection = conn; + cmd.CommandText = "UPSERT INTO accounts(id, balance) VALUES(@id1, @val1), (@id2, @val2)"; + cmd.Parameters.AddWithValue("id1", 1); + cmd.Parameters.AddWithValue("val1", 1000); + cmd.Parameters.AddWithValue("id2", 2); + cmd.Parameters.AddWithValue("val2", 250); + cmd.ExecuteNonQuery(); + } + + // Print out the balances. + System.Console.WriteLine("Initial balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + + try + { + using (var tran = conn.BeginTransaction()) + { + tran.Save("cockroach_restart"); + while (true) + { + try + { + TransferFunds(conn, tran, 1, 2, 100); + tran.Commit(); + break; + } + catch (NpgsqlException e) + { + // Check if the error code indicates a SERIALIZATION_FAILURE. + if (e.ErrorCode == 40001) + { + // Signal the database that we will attempt a retry. + tran.Rollback("cockroach_restart"); + } + else + { + throw; + } + } + } + } + } + catch (DataException e) + { + Console.WriteLine(e.Message); + } + + // Now printout the results. + Console.WriteLine("Final balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + } + } + } +} diff --git a/_includes/v20.2/app/insecure/txn-sample.go b/_includes/v20.2/app/insecure/txn-sample.go new file mode 100644 index 00000000000..2c0cd1b6da6 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.go @@ -0,0 +1,51 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + + "github.com/cockroachdb/cockroach-go/crdb" +) + +func transferFunds(tx *sql.Tx, from int, to int, amount int) error { + // Read the balance. + var fromBalance int + if err := tx.QueryRow( + "SELECT balance FROM accounts WHERE id = $1", from).Scan(&fromBalance); err != nil { + return err + } + + if fromBalance < amount { + return fmt.Errorf("insufficient funds") + } + + // Perform the transfer. + if _, err := tx.Exec( + "UPDATE accounts SET balance = balance - $1 WHERE id = $2", amount, from); err != nil { + return err + } + if _, err := tx.Exec( + "UPDATE accounts SET balance = balance + $1 WHERE id = $2", amount, to); err != nil { + return err + } + return nil +} + +func main() { + db, err := sql.Open("postgres", "postgresql://maxroach@localhost:26257/bank?sslmode=disable") + if err != nil { + log.Fatal("error connecting to the database: ", err) + } + + // Run a transfer in a transaction. + err = crdb.ExecuteTx(context.Background(), db, nil, func(tx *sql.Tx) error { + return transferFunds(tx, 1 /* from acct# */, 2 /* to acct# */, 100 /* amount */) + }) + if err == nil { + fmt.Println("Success") + } else { + log.Fatal("error: ", err) + } +} diff --git a/_includes/v20.2/app/insecure/txn-sample.js b/_includes/v20.2/app/insecure/txn-sample.js new file mode 100644 index 00000000000..c44309b01a2 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.js @@ -0,0 +1,146 @@ +var async = require('async'); +var fs = require('fs'); +var pg = require('pg'); + +// Connect to the bank database. + +var config = { + user: 'maxroach', + host: 'localhost', + database: 'bank', + port: 26257 +}; + +// Wrapper for a transaction. This automatically re-calls "op" with +// the client as an argument as long as the database server asks for +// the transaction to be retried. + +function txnWrapper(client, op, next) { + client.query('BEGIN; SAVEPOINT cockroach_restart', function (err) { + if (err) { + return next(err); + } + + var released = false; + async.doWhilst(function (done) { + var handleError = function (err) { + // If we got an error, see if it's a retryable one + // and, if so, restart. + if (err.code === '40001') { + // Signal the database that we'll retry. + return client.query('ROLLBACK TO SAVEPOINT cockroach_restart', done); + } + // A non-retryable error; break out of the + // doWhilst with an error. + return done(err); + }; + + // Attempt the work. + op(client, function (err) { + if (err) { + return handleError(err); + } + var opResults = arguments; + + // If we reach this point, release and commit. + client.query('RELEASE SAVEPOINT cockroach_restart', function (err) { + if (err) { + return handleError(err); + } + released = true; + return done.apply(null, opResults); + }); + }); + }, + function () { + return !released; + }, + function (err) { + if (err) { + client.query('ROLLBACK', function () { + next(err); + }); + } else { + var txnResults = arguments; + client.query('COMMIT', function (err) { + if (err) { + return next(err); + } else { + return next.apply(null, txnResults); + } + }); + } + }); + }); +} + +// The transaction we want to run. + +function transferFunds(client, from, to, amount, next) { + // Check the current balance. + client.query('SELECT balance FROM accounts WHERE id = $1', [from], function (err, results) { + if (err) { + return next(err); + } else if (results.rows.length === 0) { + return next(new Error('account not found in table')); + } + + var acctBal = results.rows[0].balance; + if (acctBal >= amount) { + // Perform the transfer. + async.waterfall([ + function (next) { + // Subtract amount from account 1. + client.query('UPDATE accounts SET balance = balance - $1 WHERE id = $2', [amount, from], next); + }, + function (updateResult, next) { + // Add amount to account 2. + client.query('UPDATE accounts SET balance = balance + $1 WHERE id = $2', [amount, to], next); + }, + function (updateResult, next) { + // Fetch account balances after updates. + client.query('SELECT id, balance FROM accounts', function (err, selectResult) { + next(err, selectResult ? selectResult.rows : null); + }); + } + ], next); + } else { + next(new Error('insufficient funds')); + } + }); +} + +// Create a pool. +var pool = new pg.Pool(config); + +pool.connect(function (err, client, done) { + // Closes communication with the database and exits. + var finish = function () { + done(); + process.exit(); + }; + + if (err) { + console.error('could not connect to cockroachdb', err); + finish(); + } + + // Execute the transaction. + txnWrapper(client, + function (client, next) { + transferFunds(client, 1, 2, 100, next); + }, + function (err, results) { + if (err) { + console.error('error performing transaction', err); + finish(); + } + + console.log('Balances after transfer:'); + results.forEach(function (result) { + console.log(result); + }); + + finish(); + }); +}); diff --git a/_includes/v20.2/app/insecure/txn-sample.php b/_includes/v20.2/app/insecure/txn-sample.php new file mode 100644 index 00000000000..e060d311cc3 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.php @@ -0,0 +1,71 @@ +beginTransaction(); + // This savepoint allows us to retry our transaction. + $dbh->exec("SAVEPOINT cockroach_restart"); + } catch (Exception $e) { + throw $e; + } + + while (true) { + try { + $stmt = $dbh->prepare( + 'UPDATE accounts SET balance = balance + :deposit ' . + 'WHERE id = :account AND (:deposit > 0 OR balance + :deposit >= 0)'); + + // First, withdraw the money from the old account (if possible). + $stmt->bindValue(':account', $from, PDO::PARAM_INT); + $stmt->bindValue(':deposit', -$amount, PDO::PARAM_INT); + $stmt->execute(); + if ($stmt->rowCount() == 0) { + print "source account does not exist or is underfunded\r\n"; + return; + } + + // Next, deposit into the new account (if it exists). + $stmt->bindValue(':account', $to, PDO::PARAM_INT); + $stmt->bindValue(':deposit', $amount, PDO::PARAM_INT); + $stmt->execute(); + if ($stmt->rowCount() == 0) { + print "destination account does not exist\r\n"; + return; + } + + // Attempt to release the savepoint (which is really the commit). + $dbh->exec('RELEASE SAVEPOINT cockroach_restart'); + $dbh->commit(); + return; + } catch (PDOException $e) { + if ($e->getCode() != '40001') { + // Non-recoverable error. Rollback and bubble error up the chain. + $dbh->rollBack(); + throw $e; + } else { + // Cockroach transaction retry code. Rollback to the savepoint and + // restart. + $dbh->exec('ROLLBACK TO SAVEPOINT cockroach_restart'); + } + } + } +} + +try { + $dbh = new PDO('pgsql:host=localhost;port=26257;dbname=bank;sslmode=disable', + 'maxroach', null, array( + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + )); + + transferMoney($dbh, 1, 2, 10); + + print "Account balances after transfer:\r\n"; + foreach ($dbh->query('SELECT id, balance FROM accounts') as $row) { + print $row['id'] . ': ' . $row['balance'] . "\r\n"; + } +} catch (Exception $e) { + print $e->getMessage() . "\r\n"; + exit(1); +} +?> diff --git a/_includes/v20.2/app/insecure/txn-sample.py b/_includes/v20.2/app/insecure/txn-sample.py new file mode 100644 index 00000000000..2ea05a85704 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.py @@ -0,0 +1,73 @@ +# Import the driver. +import psycopg2 +import psycopg2.errorcodes + +# Connect to the cluster. +conn = psycopg2.connect( + database='bank', + user='maxroach', + sslmode='disable', + port=26257, + host='localhost' +) + +def onestmt(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn, op): + with conn: + onestmt(conn, "SAVEPOINT cockroach_restart") + while True: + try: + # Attempt the work. + op(conn) + + # If we reach this point, commit. + onestmt(conn, "RELEASE SAVEPOINT cockroach_restart") + break + + except psycopg2.OperationalError as e: + if e.pgcode != psycopg2.errorcodes.SERIALIZATION_FAILURE: + # A non-retryable error; report this up the call stack. + raise e + # Signal the database that we'll retry. + onestmt(conn, "ROLLBACK TO SAVEPOINT cockroach_restart") + + +# The transaction we want to run. +def transfer_funds(txn, frm, to, amount): + with txn.cursor() as cur: + + # Check the current balance. + cur.execute("SELECT balance FROM accounts WHERE id = " + str(frm)) + from_balance = cur.fetchone()[0] + if from_balance < amount: + raise "Insufficient funds" + + # Perform the transfer. + cur.execute("UPDATE accounts SET balance = balance - %s WHERE id = %s", + (amount, frm)) + cur.execute("UPDATE accounts SET balance = balance + %s WHERE id = %s", + (amount, to)) + + +# Execute the transaction. +run_transaction(conn, lambda conn: transfer_funds(conn, 1, 2, 100)) + + +with conn: + with conn.cursor() as cur: + # Check account balances. + cur.execute("SELECT id, balance FROM accounts") + rows = cur.fetchall() + print('Balances after transfer:') + for row in rows: + print([str(cell) for cell in row]) + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/insecure/txn-sample.rb b/_includes/v20.2/app/insecure/txn-sample.rb new file mode 100644 index 00000000000..416efb9e24d --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.rb @@ -0,0 +1,49 @@ +# Import the driver. +require 'pg' + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn) + conn.transaction do |txn| + txn.exec('SAVEPOINT cockroach_restart') + while + begin + # Attempt the work. + yield txn + + # If we reach this point, commit. + txn.exec('RELEASE SAVEPOINT cockroach_restart') + break + rescue PG::TRSerializationFailure + txn.exec('ROLLBACK TO SAVEPOINT cockroach_restart') + end + end + end +end + +def transfer_funds(txn, from, to, amount) + txn.exec_params('SELECT balance FROM accounts WHERE id = $1', [from]) do |res| + res.each do |row| + raise 'insufficient funds' if Integer(row['balance']) < amount + end + end + txn.exec_params('UPDATE accounts SET balance = balance - $1 WHERE id = $2', [amount, from]) + txn.exec_params('UPDATE accounts SET balance = balance + $1 WHERE id = $2', [amount, to]) +end + +# Connect to the "bank" database. +conn = PG.connect( + user: 'maxroach', + dbname: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'disable' +) + +run_transaction(conn) do |txn| + transfer_funds(txn, 1, 2, 100) +end + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/insecure/txn-sample.rs b/_includes/v20.2/app/insecure/txn-sample.rs new file mode 100644 index 00000000000..d1dd0e021c9 --- /dev/null +++ b/_includes/v20.2/app/insecure/txn-sample.rs @@ -0,0 +1,60 @@ +use postgres::{error::SqlState, Client, Error, NoTls, Transaction}; + +/// Runs op inside a transaction and retries it as needed. +/// On non-retryable failures, the transaction is aborted and +/// rolled back; on success, the transaction is committed. +fn execute_txn(client: &mut Client, op: F) -> Result +where + F: Fn(&mut Transaction) -> Result, +{ + let mut txn = client.transaction()?; + loop { + let mut sp = txn.savepoint("cockroach_restart")?; + match op(&mut sp).and_then(|t| sp.commit().map(|_| t)) { + Err(ref err) + if err + .code() + .map(|e| *e == SqlState::T_R_SERIALIZATION_FAILURE) + .unwrap_or(false) => {} + r => break r, + } + } + .and_then(|t| txn.commit().map(|_| t)) +} + +fn transfer_funds(txn: &mut Transaction, from: i64, to: i64, amount: i64) -> Result<(), Error> { + // Read the balance. + let from_balance: i64 = txn + .query_one("SELECT balance FROM accounts WHERE id = $1", &[&from])? + .get(0); + + assert!(from_balance >= amount); + + // Perform the transfer. + txn.execute( + "UPDATE accounts SET balance = balance - $1 WHERE id = $2", + &[&amount, &from], + )?; + txn.execute( + "UPDATE accounts SET balance = balance + $1 WHERE id = $2", + &[&amount, &to], + )?; + Ok(()) +} + +fn main() { + let mut client = Client::connect("postgresql://maxroach@localhost:26257/bank", NoTls).unwrap(); + + // Run a transfer in a transaction. + execute_txn(&mut client, |txn| transfer_funds(txn, 1, 2, 100)).unwrap(); + + // Check account balances after the transaction. + for row in &client + .query("SELECT id, balance FROM accounts", &[]) + .unwrap() + { + let id: i64 = row.get(0); + let balance: i64 = row.get(1); + println!("{} {}", id, balance); + } +} diff --git a/_includes/v20.2/app/jooq-basic-sample/Sample.java b/_includes/v20.2/app/jooq-basic-sample/Sample.java new file mode 100644 index 00000000000..9baf2057561 --- /dev/null +++ b/_includes/v20.2/app/jooq-basic-sample/Sample.java @@ -0,0 +1,215 @@ +package com.cockroachlabs; + +import com.cockroachlabs.example.jooq.db.Tables; +import com.cockroachlabs.example.jooq.db.tables.records.AccountsRecord; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.Source; +import org.jooq.conf.RenderQuotedNames; +import org.jooq.conf.Settings; +import org.jooq.exception.DataAccessException; +import org.jooq.impl.DSL; + +import java.io.InputStream; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; + +import static com.cockroachlabs.example.jooq.db.Tables.ACCOUNTS; + +public class Sample { + + private static final Random RAND = new Random(); + private static final boolean FORCE_RETRY = false; + private static final String RETRY_SQL_STATE = "40001"; + private static final int MAX_ATTEMPT_COUNT = 6; + + private static Function addAccounts() { + return ctx -> { + long rv = 0; + + ctx.delete(ACCOUNTS).execute(); + ctx.batchInsert( + new AccountsRecord(1L, 1000L), + new AccountsRecord(2L, 250L), + new AccountsRecord(3L, 314159L) + ).execute(); + + rv = 1; + System.out.printf("APP: addAccounts() --> %d\n", rv); + return rv; + }; + } + + private static Function transferFunds(long fromId, long toId, long amount) { + return ctx -> { + long rv = 0; + + AccountsRecord fromAccount = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(fromId)); + AccountsRecord toAccount = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(toId)); + + if (!(amount > fromAccount.getBalance())) { + fromAccount.setBalance(fromAccount.getBalance() - amount); + toAccount.setBalance(toAccount.getBalance() + amount); + + ctx.batchUpdate(fromAccount, toAccount).execute(); + rv = amount; + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d\n", fromId, toId, amount, rv); + } + + return rv; + }; + } + + // Test our retry handling logic if FORCE_RETRY is true. This + // method is only used to test the retry logic. It is not + // intended for production code. + private static Function forceRetryLogic() { + return ctx -> { + long rv = -1; + try { + System.out.printf("APP: testRetryLogic: BEFORE EXCEPTION\n"); + ctx.execute("SELECT crdb_internal.force_retry('1s')"); + } catch (DataAccessException e) { + System.out.printf("APP: testRetryLogic: AFTER EXCEPTION\n"); + throw e; + } + return rv; + }; + } + + private static Function getAccountBalance(long id) { + return ctx -> { + AccountsRecord account = ctx.fetchSingle(ACCOUNTS, ACCOUNTS.ID.eq(id)); + long balance = account.getBalance(); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", id, balance); + return balance; + }; + } + + // Run SQL code in a way that automatically handles the + // transaction retry logic so we don't have to duplicate it in + // various places. + private static long runTransaction(DSLContext session, Function fn) { + AtomicLong rv = new AtomicLong(0L); + AtomicInteger attemptCount = new AtomicInteger(0); + + while (attemptCount.get() < MAX_ATTEMPT_COUNT) { + attemptCount.incrementAndGet(); + + if (attemptCount.get() > 1) { + System.out.printf("APP: Entering retry loop again, iteration %d\n", attemptCount.get()); + } + + if (session.connectionResult(connection -> { + connection.setAutoCommit(false); + System.out.printf("APP: BEGIN;\n"); + + if (attemptCount.get() == MAX_ATTEMPT_COUNT) { + String err = String.format("hit max of %s attempts, aborting", MAX_ATTEMPT_COUNT); + throw new RuntimeException(err); + } + + // This block is only used to test the retry logic. + // It is not necessary in production code. See also + // the method 'testRetryLogic()'. + if (FORCE_RETRY) { + session.fetch("SELECT now()"); + } + + try { + rv.set(fn.apply(session)); + if (rv.get() != -1) { + connection.commit(); + System.out.printf("APP: COMMIT;\n"); + return true; + } + } catch (DataAccessException | SQLException e) { + String sqlState = e instanceof SQLException ? ((SQLException) e).getSQLState() : ((DataAccessException) e).sqlState(); + + if (RETRY_SQL_STATE.equals(sqlState)) { + // Since this is a transaction retry error, we + // roll back the transaction and sleep a little + // before trying again. Each time through the + // loop we sleep for a little longer than the last + // time (A.K.A. exponential backoff). + System.out.printf("APP: retryable exception occurred:\n sql state = [%s]\n message = [%s]\n retry counter = %s\n", sqlState, e.getMessage(), attemptCount.get()); + System.out.printf("APP: ROLLBACK;\n"); + connection.rollback(); + int sleepMillis = (int)(Math.pow(2, attemptCount.get()) * 100) + RAND.nextInt(100); + System.out.printf("APP: Hit 40001 transaction retry error, sleeping %s milliseconds\n", sleepMillis); + try { + Thread.sleep(sleepMillis); + } catch (InterruptedException ignored) { + // no-op + } + rv.set(-1L); + } else { + throw e; + } + } + + return false; + })) { + break; + } + } + + return rv.get(); + } + + public static void main(String[] args) throws Exception { + try (Connection connection = DriverManager.getConnection( + "jdbc:postgresql://localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key.pk8&sslcert=certs/client.maxroach.crt", + "maxroach", + "" + )) { + DSLContext ctx = DSL.using(connection, SQLDialect.COCKROACHDB, new Settings() + .withExecuteLogging(true) + .withRenderQuotedNames(RenderQuotedNames.NEVER)); + + // Initialise database with db.sql script + try (InputStream in = Sample.class.getResourceAsStream("/db.sql")) { + ctx.parser().parse(Source.of(in).readString()).executeBatch(); + } + + long fromAccountId = 1; + long toAccountId = 2; + long transferAmount = 100; + + if (FORCE_RETRY) { + System.out.printf("APP: About to test retry logic in 'runTransaction'\n"); + runTransaction(ctx, forceRetryLogic()); + } else { + + runTransaction(ctx, addAccounts()); + long fromBalance = runTransaction(ctx, getAccountBalance(fromAccountId)); + long toBalance = runTransaction(ctx, getAccountBalance(toAccountId)); + if (fromBalance != -1 && toBalance != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalance); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalance); + } + + // Transfer $100 from account 1 to account 2 + long transferResult = runTransaction(ctx, transferFunds(fromAccountId, toAccountId, transferAmount)); + if (transferResult != -1) { + // Success! + System.out.printf("APP: transferFunds(%d, %d, %d) --> %d \n", fromAccountId, toAccountId, transferAmount, transferResult); + + long fromBalanceAfter = runTransaction(ctx, getAccountBalance(fromAccountId)); + long toBalanceAfter = runTransaction(ctx, getAccountBalance(toAccountId)); + if (fromBalanceAfter != -1 && toBalanceAfter != -1) { + // Success! + System.out.printf("APP: getAccountBalance(%d) --> %d\n", fromAccountId, fromBalanceAfter); + System.out.printf("APP: getAccountBalance(%d) --> %d\n", toAccountId, toBalanceAfter); + } + } + } + } + } +} diff --git a/_includes/v20.2/app/jooq-basic-sample/jooq-basic-sample.zip b/_includes/v20.2/app/jooq-basic-sample/jooq-basic-sample.zip new file mode 100644 index 00000000000..859305478c0 Binary files /dev/null and b/_includes/v20.2/app/jooq-basic-sample/jooq-basic-sample.zip differ diff --git a/_includes/v20.2/app/pony-basic-sample.py b/_includes/v20.2/app/pony-basic-sample.py new file mode 100644 index 00000000000..436650c7208 --- /dev/null +++ b/_includes/v20.2/app/pony-basic-sample.py @@ -0,0 +1,69 @@ +import random +from math import floor +from pony.orm import * + +db = Database() + +# The Account class corresponds to the "accounts" database table. + + +class Account(db.Entity): + _table_ = 'accounts' + id = PrimaryKey(int) + balance = Required(int) + + +db_params = dict(provider='cockroach', user='maxroach', host='localhost', port=26257, database='bank', sslmode='require', + sslrootcert='certs/ca.crt', sslkey='certs/client.maxroach.key', sslcert='certs/client.maxroach.crt') + + +sql_debug(True) # Print all generated SQL queries to stdout +db.bind(**db_params) # Bind Database object to the real database +db.generate_mapping(create_tables=True) # Create tables + + +# Store the account IDs we create for later use. + +seen_account_ids = set() + + +# The code below generates random IDs for new accounts. + +@db_session # db_session decorator manages the transactions +def create_random_accounts(n): + elems = iter(range(n)) + for i in elems: + billion = 1000000000 + new_id = floor(random.random() * billion) + seen_account_ids.add(new_id) + # Create new account + Account(id=new_id, balance=floor(random.random() * 1000000)) + + +create_random_accounts(100) + + +def get_random_account_id(): + id = random.choice(tuple(seen_account_ids)) + return id + + +@db_session(retry=10) # retry of the optimistic transaction +def transfer_funds_randomly(): + """ + Cuts a randomly selected account's balance in half, and gives the + other half to some other randomly selected account. + """ + + source_id = get_random_account_id() + sink_id = get_random_account_id() + + source = Account.get(id=source_id) + amount = floor(source.balance / 2) + + if source.balance < amount: + raise "Insufficient funds" + + source.balance -= amount + sink = Account.get(id=sink_id) + sink.balance += amount diff --git a/_includes/v20.2/app/project.clj b/_includes/v20.2/app/project.clj new file mode 100644 index 00000000000..41efc324b59 --- /dev/null +++ b/_includes/v20.2/app/project.clj @@ -0,0 +1,7 @@ +(defproject test "0.1" + :description "CockroachDB test" + :url "http://cockroachlabs.com/" + :dependencies [[org.clojure/clojure "1.8.0"] + [org.clojure/java.jdbc "0.6.1"] + [org.postgresql/postgresql "9.4.1211"]] + :main test.test) diff --git a/_includes/v20.2/app/retry-errors.md b/_includes/v20.2/app/retry-errors.md new file mode 100644 index 00000000000..5f219f53e12 --- /dev/null +++ b/_includes/v20.2/app/retry-errors.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +Your application should [use a retry loop to handle transaction errors](error-handling-and-troubleshooting.html#transaction-retry-errors) that can occur under contention. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/app/see-also-links.md b/_includes/v20.2/app/see-also-links.md new file mode 100644 index 00000000000..e5dd6173c99 --- /dev/null +++ b/_includes/v20.2/app/see-also-links.md @@ -0,0 +1,9 @@ +You might also be interested in the following pages: + +- [Client Connection Parameters](connection-parameters.html) +- [Data Replication](demo-data-replication.html) +- [Fault Tolerance & Recovery](demo-fault-tolerance-and-recovery.html) +- [Automatic Rebalancing](demo-automatic-rebalancing.html) +- [Cross-Cloud Migration](demo-automatic-cloud-migration.html) +- [Follow-the-Workload](demo-follow-the-workload.html) +- [Automated Operations](orchestrate-a-local-cluster-with-kubernetes-insecure.html) diff --git a/_includes/v20.2/app/sequelize-basic-sample.js b/_includes/v20.2/app/sequelize-basic-sample.js new file mode 100644 index 00000000000..d87ff2ca5a5 --- /dev/null +++ b/_includes/v20.2/app/sequelize-basic-sample.js @@ -0,0 +1,62 @@ +var Sequelize = require('sequelize-cockroachdb'); +var fs = require('fs'); + +// Connect to CockroachDB through Sequelize. +var sequelize = new Sequelize('bank', 'maxroach', '', { + dialect: 'postgres', + port: 26257, + logging: false, + dialectOptions: { + ssl: { + ca: fs.readFileSync('certs/ca.crt') + .toString(), + key: fs.readFileSync('certs/client.maxroach.key') + .toString(), + cert: fs.readFileSync('certs/client.maxroach.crt') + .toString() + } + } +}); + +// Define the Account model for the "accounts" table. +var Account = sequelize.define('accounts', { + id: { + type: Sequelize.INTEGER, + primaryKey: true + }, + balance: { + type: Sequelize.INTEGER + } +}); + +// Create the "accounts" table. +Account.sync({ + force: true + }) + .then(function () { + // Insert two rows into the "accounts" table. + return Account.bulkCreate([{ + id: 1, + balance: 1000 + }, + { + id: 2, + balance: 250 + } + ]); + }) + .then(function () { + // Retrieve accounts. + return Account.findAll(); + }) + .then(function (accounts) { + // Print out the balances. + accounts.forEach(function (account) { + console.log(account.id + ' ' + account.balance); + }); + process.exit(0); + }) + .catch(function (err) { + console.error('error: ' + err.message); + process.exit(1); + }); diff --git a/_includes/v20.2/app/sqlalchemy-basic-sample.py b/_includes/v20.2/app/sqlalchemy-basic-sample.py new file mode 100644 index 00000000000..6fa27d5691f --- /dev/null +++ b/_includes/v20.2/app/sqlalchemy-basic-sample.py @@ -0,0 +1,110 @@ +import random +from math import floor +from sqlalchemy import create_engine, Column, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from cockroachdb.sqlalchemy import run_transaction + +Base = declarative_base() + + +# The Account class corresponds to the "accounts" database table. +class Account(Base): + __tablename__ = 'accounts' + id = Column(Integer, primary_key=True) + balance = Column(Integer) + + +# Create an engine to communicate with the database. The +# "cockroachdb://" prefix for the engine URL indicates that we are +# connecting to CockroachDB using the 'cockroachdb' dialect. +# For more information, see +# https://github.com/cockroachdb/sqlalchemy-cockroachdb. + +secure_cluster = True # Set to False for insecure clusters +connect_args = {} + +if secure_cluster: + connect_args = { + 'sslmode': 'require', + 'sslrootcert': 'certs/ca.crt', + 'sslkey': 'certs/client.maxroach.key', + 'sslcert': 'certs/client.maxroach.crt' + } +else: + connect_args = {'sslmode': 'disable'} + +engine = create_engine( + 'cockroachdb://maxroach@localhost:26257/bank', + connect_args=connect_args, + echo=True # Log SQL queries to stdout +) + +# Automatically create the "accounts" table based on the Account class. +Base.metadata.create_all(engine) + + +# Store the account IDs we create for later use. + +seen_account_ids = set() + + +# The code below generates random IDs for new accounts. + +def create_random_accounts(sess, n): + """Create N new accounts with random IDs and random account balances. + + Note that since this is a demo, we don't do any work to ensure the + new IDs don't collide with existing IDs. + """ + new_accounts = [] + elems = iter(range(n)) + for i in elems: + billion = 1000000000 + new_id = floor(random.random()*billion) + seen_account_ids.add(new_id) + new_accounts.append( + Account( + id=new_id, + balance=floor(random.random()*1000000) + ) + ) + sess.add_all(new_accounts) + + +run_transaction(sessionmaker(bind=engine), + lambda s: create_random_accounts(s, 100)) + + +# Helper for getting random existing account IDs. + +def get_random_account_id(): + id = random.choice(tuple(seen_account_ids)) + return id + + +def transfer_funds_randomly(session): + """Transfer money randomly between accounts (during SESSION). + + Cuts a randomly selected account's balance in half, and gives the + other half to some other randomly selected account. + """ + source_id = get_random_account_id() + sink_id = get_random_account_id() + + source = session.query(Account).filter_by(id=source_id).one() + amount = floor(source.balance/2) + + # Check balance of the first account. + if source.balance < amount: + raise "Insufficient funds" + + source.balance -= amount + session.query(Account).filter_by(id=sink_id).update( + {"balance": (Account.balance + amount)} + ) + + +# Run the transfer inside a transaction. + +run_transaction(sessionmaker(bind=engine), transfer_funds_randomly) diff --git a/_includes/v20.2/app/sqlalchemy-large-txns.py b/_includes/v20.2/app/sqlalchemy-large-txns.py new file mode 100644 index 00000000000..bc7399b663c --- /dev/null +++ b/_includes/v20.2/app/sqlalchemy-large-txns.py @@ -0,0 +1,60 @@ +from sqlalchemy import create_engine, Column, Float, Integer +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from cockroachdb.sqlalchemy import run_transaction +from random import random + +Base = declarative_base() + +# The code below assumes you are running as 'root' and have run +# the following SQL statements against an insecure cluster. + +# CREATE DATABASE pointstore; + +# USE pointstore; + +# CREATE TABLE points ( +# id INT PRIMARY KEY DEFAULT unique_rowid(), +# x FLOAT NOT NULL, +# y FLOAT NOT NULL, +# z FLOAT NOT NULL +# ); + +engine = create_engine( + 'cockroachdb://root@localhost:26257/pointstore', + connect_args={ + 'sslmode': 'disable', + }, + echo=True +) + + +class Point(Base): + __tablename__ = 'points' + id = Column(Integer, primary_key=True) + x = Column(Float) + y = Column(Float) + z = Column(Float) + + +def add_points(num_points): + chunk_size = 1000 # Tune this based on object sizes. + + def add_points_helper(sess, chunk, num_points): + points = [] + for i in range(chunk, min(chunk + chunk_size, num_points)): + points.append( + Point(x=random()*1024, y=random()*1024, z=random()*1024) + ) + sess.bulk_save_objects(points) + + for chunk in range(0, num_points, chunk_size): + run_transaction( + sessionmaker(bind=engine), + lambda s: add_points_helper( + s, chunk, min(chunk + chunk_size, num_points) + ) + ) + + +add_points(10000) diff --git a/_includes/v20.2/app/txn-sample.clj b/_includes/v20.2/app/txn-sample.clj new file mode 100644 index 00000000000..c093078ebc4 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.clj @@ -0,0 +1,48 @@ +(ns test.test + (:require [clojure.java.jdbc :as j] + [test.util :as util])) + +;; Define the connection parameters to the cluster. +(def db-spec {:dbtype "postgresql" + :dbname "bank" + :host "localhost" + :port "26257" + :ssl true + :sslmode "require" + :sslcert "certs/client.maxroach.crt" + :sslkey "certs/client.maxroach.key.pk8" + :user "maxroach"}) + +;; The transaction we want to run. +(defn transferFunds + [txn from to amount] + + ;; Check the current balance. + (let [fromBalance (->> (j/query txn ["SELECT balance FROM accounts WHERE id = ?" from]) + (mapv :balance) + (first))] + (when (< fromBalance amount) + (throw (Exception. "Insufficient funds")))) + + ;; Perform the transfer. + (j/execute! txn [(str "UPDATE accounts SET balance = balance - " amount " WHERE id = " from)]) + (j/execute! txn [(str "UPDATE accounts SET balance = balance + " amount " WHERE id = " to)])) + +(defn test-txn [] + ;; Connect to the cluster and run the code below with + ;; the connection object bound to 'conn'. + (j/with-db-connection [conn db-spec] + + ;; Execute the transaction within an automatic retry block; + ;; the transaction object is bound to 'txn'. + (util/with-txn-retry [txn conn] + (transferFunds txn 1 2 100)) + + ;; Execute a query outside of an automatic retry block. + (println "Balances after transfer:") + (->> (j/query conn ["SELECT id, balance FROM accounts"]) + (map println) + (doall)))) + +(defn -main [& args] + (test-txn)) diff --git a/_includes/v20.2/app/txn-sample.cpp b/_includes/v20.2/app/txn-sample.cpp new file mode 100644 index 00000000000..728e4a2e5cc --- /dev/null +++ b/_includes/v20.2/app/txn-sample.cpp @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; + +void transferFunds( + pqxx::dbtransaction *tx, int from, int to, int amount) { + // Read the balance. + pqxx::result r = tx->exec( + "SELECT balance FROM accounts WHERE id = " + to_string(from)); + assert(r.size() == 1); + int fromBalance = r[0][0].as(); + + if (fromBalance < amount) { + throw domain_error("insufficient funds"); + } + + // Perform the transfer. + tx->exec("UPDATE accounts SET balance = balance - " + + to_string(amount) + " WHERE id = " + to_string(from)); + tx->exec("UPDATE accounts SET balance = balance + " + + to_string(amount) + " WHERE id = " + to_string(to)); +} + + +// ExecuteTx runs fn inside a transaction and retries it as needed. +// On non-retryable failures, the transaction is aborted and rolled +// back; on success, the transaction is committed. +// +// For more information about CockroachDB's transaction model see +// https://cockroachlabs.com/docs/transactions.html. +// +// NOTE: the supplied exec closure should not have external side +// effects beyond changes to the database. +void executeTx( + pqxx::connection *c, function fn) { + pqxx::work tx(*c); + while (true) { + try { + pqxx::subtransaction s(tx, "cockroach_restart"); + fn(&s); + s.commit(); + break; + } catch (const pqxx::pqxx_exception& e) { + // Swallow "transaction restart" errors; the transaction will be retried. + // Unfortunately libpqxx doesn't give us access to the error code, so we + // do string matching to identify retryable errors. + if (string(e.base().what()).find("restart transaction:") == string::npos) { + throw; + } + } + } + tx.commit(); +} + +int main() { + try { + pqxx::connection c("dbname=bank user=maxroach sslmode=require sslkey=certs/client.maxroach.key sslcert=certs/client.maxroach.crt port=26257 host=localhost"); + + executeTx(&c, [](pqxx::dbtransaction *tx) { + transferFunds(tx, 1, 2, 100); + }); + } + catch (const exception &e) { + cerr << e.what() << endl; + return 1; + } + cout << "Success" << endl; + return 0; +} diff --git a/_includes/v20.2/app/txn-sample.cs b/_includes/v20.2/app/txn-sample.cs new file mode 100644 index 00000000000..ced5063a4b9 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.cs @@ -0,0 +1,168 @@ +using System; +using System.Data; +using System.Security.Cryptography.X509Certificates; +using System.Net.Security; +using Npgsql; + +namespace Cockroach +{ + class MainClass + { + static void Main(string[] args) + { + var connStringBuilder = new NpgsqlConnectionStringBuilder(); + connStringBuilder.Host = "localhost"; + connStringBuilder.Port = 26257; + connStringBuilder.SslMode = SslMode.Require; + connStringBuilder.Username = "maxroach"; + connStringBuilder.Database = "bank"; + TxnSample(connStringBuilder.ConnectionString); + } + + static void TransferFunds(NpgsqlConnection conn, NpgsqlTransaction tran, int from, int to, int amount) + { + int balance = 0; + using (var cmd = new NpgsqlCommand(String.Format("SELECT balance FROM accounts WHERE id = {0}", from), conn, tran)) + using (var reader = cmd.ExecuteReader()) + { + if (reader.Read()) + { + balance = reader.GetInt32(0); + } + else + { + throw new DataException(String.Format("Account id={0} not found", from)); + } + } + if (balance < amount) + { + throw new DataException(String.Format("Insufficient balance in account id={0}", from)); + } + using (var cmd = new NpgsqlCommand(String.Format("UPDATE accounts SET balance = balance - {0} where id = {1}", amount, from), conn, tran)) + { + cmd.ExecuteNonQuery(); + } + using (var cmd = new NpgsqlCommand(String.Format("UPDATE accounts SET balance = balance + {0} where id = {1}", amount, to), conn, tran)) + { + cmd.ExecuteNonQuery(); + } + } + + static void TxnSample(string connString) + { + using (var conn = new NpgsqlConnection(connString)) + { + conn.ProvideClientCertificatesCallback += ProvideClientCertificatesCallback; + conn.UserCertificateValidationCallback += UserCertificateValidationCallback; + + conn.Open(); + + // Create the "accounts" table. + new NpgsqlCommand("CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT)", conn).ExecuteNonQuery(); + + // Insert two rows into the "accounts" table. + using (var cmd = new NpgsqlCommand()) + { + cmd.Connection = conn; + cmd.CommandText = "UPSERT INTO accounts(id, balance) VALUES(@id1, @val1), (@id2, @val2)"; + cmd.Parameters.AddWithValue("id1", 1); + cmd.Parameters.AddWithValue("val1", 1000); + cmd.Parameters.AddWithValue("id2", 2); + cmd.Parameters.AddWithValue("val2", 250); + cmd.ExecuteNonQuery(); + } + + // Print out the balances. + System.Console.WriteLine("Initial balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + + try + { + using (var tran = conn.BeginTransaction()) + { + tran.Save("cockroach_restart"); + while (true) + { + try + { + TransferFunds(conn, tran, 1, 2, 100); + tran.Commit(); + break; + } + catch (NpgsqlException e) + { + // Check if the error code indicates a SERIALIZATION_FAILURE. + if (e.ErrorCode == 40001) + { + // Signal the database that we will attempt a retry. + tran.Rollback("cockroach_restart"); + } + else + { + throw; + } + } + } + } + } + catch (DataException e) + { + Console.WriteLine(e.Message); + } + + // Now printout the results. + Console.WriteLine("Final balances:"); + using (var cmd = new NpgsqlCommand("SELECT id, balance FROM accounts", conn)) + using (var reader = cmd.ExecuteReader()) + while (reader.Read()) + Console.Write("\taccount {0}: {1}\n", reader.GetValue(0), reader.GetValue(1)); + } + } + + static void ProvideClientCertificatesCallback(X509CertificateCollection clientCerts) + { + // To be able to add a certificate with a private key included, we must convert it to + // a PKCS #12 format. The following openssl command does this: + // openssl pkcs12 -inkey client.maxroach.key -in client.maxroach.crt -export -out client.maxroach.pfx + // As of 2018-12-10, you need to provide a password for this to work on macOS. + // See https://github.com/dotnet/corefx/issues/24225 + clientCerts.Add(new X509Certificate2("client.maxroach.pfx", "pass")); + } + + // By default, .Net does all of its certificate verification using the system certificate store. + // This callback is necessary to validate the server certificate against a CA certificate file. + static bool UserCertificateValidationCallback(object sender, X509Certificate certificate, X509Chain defaultChain, SslPolicyErrors defaultErrors) + { + X509Certificate2 caCert = new X509Certificate2("ca.crt"); + X509Chain caCertChain = new X509Chain(); + caCertChain.ChainPolicy = new X509ChainPolicy() + { + RevocationMode = X509RevocationMode.NoCheck, + RevocationFlag = X509RevocationFlag.EntireChain + }; + caCertChain.ChainPolicy.ExtraStore.Add(caCert); + + X509Certificate2 serverCert = new X509Certificate2(certificate); + + caCertChain.Build(serverCert); + if (caCertChain.ChainStatus.Length == 0) + { + // No errors + return true; + } + + foreach (X509ChainStatus status in caCertChain.ChainStatus) + { + // Check if we got any errors other than UntrustedRoot (which we will always get if we don't install the CA cert to the system store) + if (status.Status != X509ChainStatusFlags.UntrustedRoot) + { + return false; + } + } + return true; + } + } +} diff --git a/_includes/v20.2/app/txn-sample.go b/_includes/v20.2/app/txn-sample.go new file mode 100644 index 00000000000..fc15275abca --- /dev/null +++ b/_includes/v20.2/app/txn-sample.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "log" + + "github.com/cockroachdb/cockroach-go/crdb" +) + +func transferFunds(tx *sql.Tx, from int, to int, amount int) error { + // Read the balance. + var fromBalance int + if err := tx.QueryRow( + "SELECT balance FROM accounts WHERE id = $1", from).Scan(&fromBalance); err != nil { + return err + } + + if fromBalance < amount { + return fmt.Errorf("insufficient funds") + } + + // Perform the transfer. + if _, err := tx.Exec( + "UPDATE accounts SET balance = balance - $1 WHERE id = $2", amount, from); err != nil { + return err + } + if _, err := tx.Exec( + "UPDATE accounts SET balance = balance + $1 WHERE id = $2", amount, to); err != nil { + return err + } + return nil +} + +func main() { + db, err := sql.Open("postgres", + "postgresql://maxroach@localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key&sslcert=certs/client.maxroach.crt") + if err != nil { + log.Fatal("error connecting to the database: ", err) + } + defer db.Close() + + // Run a transfer in a transaction. + err = crdb.ExecuteTx(context.Background(), db, nil, func(tx *sql.Tx) error { + return transferFunds(tx, 1 /* from acct# */, 2 /* to acct# */, 100 /* amount */) + }) + if err == nil { + fmt.Println("Success") + } else { + log.Fatal("error: ", err) + } +} diff --git a/_includes/v20.2/app/txn-sample.js b/_includes/v20.2/app/txn-sample.js new file mode 100644 index 00000000000..1eebaacad30 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.js @@ -0,0 +1,154 @@ +var async = require('async'); +var fs = require('fs'); +var pg = require('pg'); + +// Connect to the bank database. + +var config = { + user: 'maxroach', + host: 'localhost', + database: 'bank', + port: 26257, + ssl: { + ca: fs.readFileSync('certs/ca.crt') + .toString(), + key: fs.readFileSync('certs/client.maxroach.key') + .toString(), + cert: fs.readFileSync('certs/client.maxroach.crt') + .toString() + } +}; + +// Wrapper for a transaction. This automatically re-calls "op" with +// the client as an argument as long as the database server asks for +// the transaction to be retried. + +function txnWrapper(client, op, next) { + client.query('BEGIN; SAVEPOINT cockroach_restart', function (err) { + if (err) { + return next(err); + } + + var released = false; + async.doWhilst(function (done) { + var handleError = function (err) { + // If we got an error, see if it's a retryable one + // and, if so, restart. + if (err.code === '40001') { + // Signal the database that we'll retry. + return client.query('ROLLBACK TO SAVEPOINT cockroach_restart', done); + } + // A non-retryable error; break out of the + // doWhilst with an error. + return done(err); + }; + + // Attempt the work. + op(client, function (err) { + if (err) { + return handleError(err); + } + var opResults = arguments; + + // If we reach this point, release and commit. + client.query('RELEASE SAVEPOINT cockroach_restart', function (err) { + if (err) { + return handleError(err); + } + released = true; + return done.apply(null, opResults); + }); + }); + }, + function () { + return !released; + }, + function (err) { + if (err) { + client.query('ROLLBACK', function () { + next(err); + }); + } else { + var txnResults = arguments; + client.query('COMMIT', function (err) { + if (err) { + return next(err); + } else { + return next.apply(null, txnResults); + } + }); + } + }); + }); +} + +// The transaction we want to run. + +function transferFunds(client, from, to, amount, next) { + // Check the current balance. + client.query('SELECT balance FROM accounts WHERE id = $1', [from], function (err, results) { + if (err) { + return next(err); + } else if (results.rows.length === 0) { + return next(new Error('account not found in table')); + } + + var acctBal = results.rows[0].balance; + if (acctBal >= amount) { + // Perform the transfer. + async.waterfall([ + function (next) { + // Subtract amount from account 1. + client.query('UPDATE accounts SET balance = balance - $1 WHERE id = $2', [amount, from], next); + }, + function (updateResult, next) { + // Add amount to account 2. + client.query('UPDATE accounts SET balance = balance + $1 WHERE id = $2', [amount, to], next); + }, + function (updateResult, next) { + // Fetch account balances after updates. + client.query('SELECT id, balance FROM accounts', function (err, selectResult) { + next(err, selectResult ? selectResult.rows : null); + }); + } + ], next); + } else { + next(new Error('insufficient funds')); + } + }); +} + +// Create a pool. +var pool = new pg.Pool(config); + +pool.connect(function (err, client, done) { + // Closes communication with the database and exits. + var finish = function () { + done(); + process.exit(); + }; + + if (err) { + console.error('could not connect to cockroachdb', err); + finish(); + } + + // Execute the transaction. + txnWrapper(client, + function (client, next) { + transferFunds(client, 1, 2, 100, next); + }, + function (err, results) { + if (err) { + console.error('error performing transaction', err); + finish(); + } + + console.log('Balances after transfer:'); + results.forEach(function (result) { + console.log(result); + }); + + finish(); + }); +}); diff --git a/_includes/v20.2/app/txn-sample.php b/_includes/v20.2/app/txn-sample.php new file mode 100644 index 00000000000..363dbcd73cd --- /dev/null +++ b/_includes/v20.2/app/txn-sample.php @@ -0,0 +1,71 @@ +beginTransaction(); + // This savepoint allows us to retry our transaction. + $dbh->exec("SAVEPOINT cockroach_restart"); + } catch (Exception $e) { + throw $e; + } + + while (true) { + try { + $stmt = $dbh->prepare( + 'UPDATE accounts SET balance = balance + :deposit ' . + 'WHERE id = :account AND (:deposit > 0 OR balance + :deposit >= 0)'); + + // First, withdraw the money from the old account (if possible). + $stmt->bindValue(':account', $from, PDO::PARAM_INT); + $stmt->bindValue(':deposit', -$amount, PDO::PARAM_INT); + $stmt->execute(); + if ($stmt->rowCount() == 0) { + print "source account does not exist or is underfunded\r\n"; + return; + } + + // Next, deposit into the new account (if it exists). + $stmt->bindValue(':account', $to, PDO::PARAM_INT); + $stmt->bindValue(':deposit', $amount, PDO::PARAM_INT); + $stmt->execute(); + if ($stmt->rowCount() == 0) { + print "destination account does not exist\r\n"; + return; + } + + // Attempt to release the savepoint (which is really the commit). + $dbh->exec('RELEASE SAVEPOINT cockroach_restart'); + $dbh->commit(); + return; + } catch (PDOException $e) { + if ($e->getCode() != '40001') { + // Non-recoverable error. Rollback and bubble error up the chain. + $dbh->rollBack(); + throw $e; + } else { + // Cockroach transaction retry code. Rollback to the savepoint and + // restart. + $dbh->exec('ROLLBACK TO SAVEPOINT cockroach_restart'); + } + } + } +} + +try { + $dbh = new PDO('pgsql:host=localhost;port=26257;dbname=bank;sslmode=require;sslrootcert=certs/ca.crt;sslkey=certs/client.maxroach.key;sslcert=certs/client.maxroach.crt', + 'maxroach', null, array( + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + )); + + transferMoney($dbh, 1, 2, 10); + + print "Account balances after transfer:\r\n"; + foreach ($dbh->query('SELECT id, balance FROM accounts') as $row) { + print $row['id'] . ': ' . $row['balance'] . "\r\n"; + } +} catch (Exception $e) { + print $e->getMessage() . "\r\n"; + exit(1); +} +?> diff --git a/_includes/v20.2/app/txn-sample.py b/_includes/v20.2/app/txn-sample.py new file mode 100644 index 00000000000..d4c86a36cc8 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.py @@ -0,0 +1,76 @@ +# Import the driver. +import psycopg2 +import psycopg2.errorcodes + +# Connect to the cluster. +conn = psycopg2.connect( + database='bank', + user='maxroach', + sslmode='require', + sslrootcert='certs/ca.crt', + sslkey='certs/client.maxroach.key', + sslcert='certs/client.maxroach.crt', + port=26257, + host='localhost' +) + +def onestmt(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn, op): + with conn: + onestmt(conn, "SAVEPOINT cockroach_restart") + while True: + try: + # Attempt the work. + op(conn) + + # If we reach this point, commit. + onestmt(conn, "RELEASE SAVEPOINT cockroach_restart") + break + + except psycopg2.OperationalError as e: + if e.pgcode != psycopg2.errorcodes.SERIALIZATION_FAILURE: + # A non-retryable error; report this up the call stack. + raise e + # Signal the database that we'll retry. + onestmt(conn, "ROLLBACK TO SAVEPOINT cockroach_restart") + + +# The transaction we want to run. +def transfer_funds(txn, frm, to, amount): + with txn.cursor() as cur: + + # Check the current balance. + cur.execute("SELECT balance FROM accounts WHERE id = " + str(frm)) + from_balance = cur.fetchone()[0] + if from_balance < amount: + raise "Insufficient funds" + + # Perform the transfer. + cur.execute("UPDATE accounts SET balance = balance - %s WHERE id = %s", + (amount, frm)) + cur.execute("UPDATE accounts SET balance = balance + %s WHERE id = %s", + (amount, to)) + + +# Execute the transaction. +run_transaction(conn, lambda conn: transfer_funds(conn, 1, 2, 100)) + + +with conn: + with conn.cursor() as cur: + # Check account balances. + cur.execute("SELECT id, balance FROM accounts") + rows = cur.fetchall() + print('Balances after transfer:') + for row in rows: + print([str(cell) for cell in row]) + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/txn-sample.rb b/_includes/v20.2/app/txn-sample.rb new file mode 100644 index 00000000000..1c3e028fdf7 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.rb @@ -0,0 +1,52 @@ +# Import the driver. +require 'pg' + +# Wrapper for a transaction. +# This automatically re-calls "op" with the open transaction as an argument +# as long as the database server asks for the transaction to be retried. +def run_transaction(conn) + conn.transaction do |txn| + txn.exec('SAVEPOINT cockroach_restart') + while + begin + # Attempt the work. + yield txn + + # If we reach this point, commit. + txn.exec('RELEASE SAVEPOINT cockroach_restart') + break + rescue PG::TRSerializationFailure + txn.exec('ROLLBACK TO SAVEPOINT cockroach_restart') + end + end + end +end + +def transfer_funds(txn, from, to, amount) + txn.exec_params('SELECT balance FROM accounts WHERE id = $1', [from]) do |res| + res.each do |row| + raise 'insufficient funds' if Integer(row['balance']) < amount + end + end + txn.exec_params('UPDATE accounts SET balance = balance - $1 WHERE id = $2', [amount, from]) + txn.exec_params('UPDATE accounts SET balance = balance + $1 WHERE id = $2', [amount, to]) +end + +# Connect to the "bank" database. +conn = PG.connect( + user: 'maxroach', + dbname: 'bank', + host: 'localhost', + port: 26257, + sslmode: 'require', + sslrootcert: 'certs/ca.crt', + sslkey:'certs/client.maxroach.key', + sslcert:'certs/client.maxroach.crt' +) + +run_transaction(conn) do |txn| + transfer_funds(txn, 1, 2, 100) +end + +# Close communication with the database. +conn.close() diff --git a/_includes/v20.2/app/txn-sample.rs b/_includes/v20.2/app/txn-sample.rs new file mode 100644 index 00000000000..c8e099b89e6 --- /dev/null +++ b/_includes/v20.2/app/txn-sample.rs @@ -0,0 +1,73 @@ +use openssl::error::ErrorStack; +use openssl::ssl::{SslConnector, SslFiletype, SslMethod}; +use postgres::{error::SqlState, Client, Error, Transaction}; +use postgres_openssl::MakeTlsConnector; + +/// Runs op inside a transaction and retries it as needed. +/// On non-retryable failures, the transaction is aborted and +/// rolled back; on success, the transaction is committed. +fn execute_txn(client: &mut Client, op: F) -> Result +where + F: Fn(&mut Transaction) -> Result, +{ + let mut txn = client.transaction()?; + loop { + let mut sp = txn.savepoint("cockroach_restart")?; + match op(&mut sp).and_then(|t| sp.commit().map(|_| t)) { + Err(ref err) + if err + .code() + .map(|e| *e == SqlState::T_R_SERIALIZATION_FAILURE) + .unwrap_or(false) => {} + r => break r, + } + } + .and_then(|t| txn.commit().map(|_| t)) +} + +fn transfer_funds(txn: &mut Transaction, from: i64, to: i64, amount: i64) -> Result<(), Error> { + // Read the balance. + let from_balance: i64 = txn + .query_one("SELECT balance FROM accounts WHERE id = $1", &[&from])? + .get(0); + + assert!(from_balance >= amount); + + // Perform the transfer. + txn.execute( + "UPDATE accounts SET balance = balance - $1 WHERE id = $2", + &[&amount, &from], + )?; + txn.execute( + "UPDATE accounts SET balance = balance + $1 WHERE id = $2", + &[&amount, &to], + )?; + Ok(()) +} + +fn ssl_config() -> Result { + let mut builder = SslConnector::builder(SslMethod::tls())?; + builder.set_ca_file("certs/ca.crt")?; + builder.set_certificate_chain_file("certs/client.maxroach.crt")?; + builder.set_private_key_file("certs/client.maxroach.key", SslFiletype::PEM)?; + Ok(MakeTlsConnector::new(builder.build())) +} + +fn main() { + let connector = ssl_config().unwrap(); + let mut client = + Client::connect("postgresql://maxroach@localhost:26257/bank", connector).unwrap(); + + // Run a transfer in a transaction. + execute_txn(&mut client, |txn| transfer_funds(txn, 1, 2, 100)).unwrap(); + + // Check account balances after the transaction. + for row in &client + .query("SELECT id, balance FROM accounts", &[]) + .unwrap() + { + let id: i64 = row.get(0); + let balance: i64 = row.get(1); + println!("{} {}", id, balance); + } +} diff --git a/_includes/v20.2/app/util.clj b/_includes/v20.2/app/util.clj new file mode 100644 index 00000000000..d040affe794 --- /dev/null +++ b/_includes/v20.2/app/util.clj @@ -0,0 +1,38 @@ +(ns test.util + (:require [clojure.java.jdbc :as j] + [clojure.walk :as walk])) + +(defn txn-restart-err? + "Takes an exception and returns true if it is a CockroachDB retry error." + [e] + (when-let [m (.getMessage e)] + (condp instance? e + java.sql.BatchUpdateException + (and (re-find #"getNextExc" m) + (txn-restart-err? (.getNextException e))) + + org.postgresql.util.PSQLException + (= (.getSQLState e) "40001") ; 40001 is the code returned by CockroachDB retry errors. + + false))) + +;; Wrapper for a transaction. +;; This automatically invokes the body again as long as the database server +;; asks the transaction to be retried. + +(defmacro with-txn-retry + "Wrap an evaluation within a CockroachDB retry block." + [[txn c] & body] + `(j/with-db-transaction [~txn ~c] + (loop [] + (j/execute! ~txn ["savepoint cockroach_restart"]) + (let [res# (try (let [r# (do ~@body)] + {:ok r#}) + (catch java.sql.SQLException e# + (if (txn-restart-err? e#) + {:retry true} + (throw e#))))] + (if (:retry res#) + (do (j/execute! ~txn ["rollback to savepoint cockroach_restart"]) + (recur)) + (:ok res#)))))) diff --git a/_includes/v20.2/backups/advanced-examples-list.md b/_includes/v20.2/backups/advanced-examples-list.md new file mode 100644 index 00000000000..a4ccb450002 --- /dev/null +++ b/_includes/v20.2/backups/advanced-examples-list.md @@ -0,0 +1,9 @@ +For examples of advanced `BACKUP` and `RESTORE` use cases, see [Back up and Restore Data - Advanced Options](backup-and-restore-advanced-options.html). Advanced examples include: + +- [Incremental backups with a specified destination](backup-and-restore-advanced-options.html#incremental-backups-with-explicitly-specified-destinations) +- [Backup with revision history and point-in-time restore](backup-and-restore-advanced-options.html#backup-with-revision-history-and-point-in-time-restore) +- [Locality-aware backup and restore](backup-and-restore-advanced-options.html#locality-aware-backup-and-restore) +- [Encrypted backup and restore](backup-and-restore-advanced-options.html#encrypted-backup-and-restore) +- [Restore into a different database](backup-and-restore-advanced-options.html#restore-into-a-different-database) +- [Remove the foreign key before restore](backup-and-restore-advanced-options.html#remove-the-foreign-key-before-restore) +- [Restoring users from `system.users` backup](backup-and-restore-advanced-options.html#restoring-users-from-system-users-backup) diff --git a/_includes/v20.2/backups/encrypted-backup-description.md b/_includes/v20.2/backups/encrypted-backup-description.md new file mode 100644 index 00000000000..f96843d623b --- /dev/null +++ b/_includes/v20.2/backups/encrypted-backup-description.md @@ -0,0 +1,11 @@ + You can encrypt full or incremental backups by using the [`encryption_passphrase` option](backup.html#with-encryption-passphrase). Files written by the backup (including `BACKUP` manifests and data files) are encrypted using the specified passphrase to derive a key. To restore the encrypted backup, the same `encryption_passphrase` option (with the same passphrase) must included in the [`RESTORE`](restore.html) statement. + +When used with [incremental backups](backup.html#incremental-backups), the `encryption_passphrase` option is applied to all the [backup file URLs](backup.html#backup-file-urls), which means the same passphrase must be used when appending another incremental backup to an existing backup. Similarly, when used with [locality-aware backups](backup-and-restore-advanced-options.html#locality-aware-backup-and-restore), the passphrase provided is applied to files in all localities. + +Encryption is done using [AES-256-GCM](https://en.wikipedia.org/wiki/Galois/Counter_Mode), and GCM is used to both encrypt and authenticate the files. A random [salt](https://en.wikipedia.org/wiki/Salt_(cryptography)) is used to derive a once-per-backup [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) key from the specified passphrase, and then a random [initialization vector](https://en.wikipedia.org/wiki/Initialization_vector) is used per-file. CockroachDB uses [PBKDF2](https://en.wikipedia.org/wiki/PBKDF2) with 64,000 iterations for the key derivation. + +{{site.data.alerts.callout_info}} +`BACKUP` and `RESTORE` will use more memory when using encryption, as both the plain-text and cipher-text of a given file are held in memory during encryption and decryption. +{{site.data.alerts.end}} + +For an example of an encrypted backup, see [Create an encrypted backup](backup-and-restore-advanced-options.html#create-an-encrypted-backup). diff --git a/_includes/v20.2/cdc/core-csv.md b/_includes/v20.2/cdc/core-csv.md new file mode 100644 index 00000000000..4ee6bfc587d --- /dev/null +++ b/_includes/v20.2/cdc/core-csv.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +To determine how wide the columns need to be, the default `table` display format in `cockroach sql` buffers the results it receives from the server before printing them to the console. When consuming core changefeed data using `cockroach sql`, it's important to use a display format like `csv` that does not buffer its results. To set the display format, use the [`--format=csv` flag](cockroach-sql.html#sql-flag-format) when starting the [built-in SQL client](cockroach-sql.html), or set the [`\set display_format=csv` option](cockroach-sql.html#client-side-options) once the SQL client is open. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/cdc/core-url.md b/_includes/v20.2/cdc/core-url.md new file mode 100644 index 00000000000..7241e203aa7 --- /dev/null +++ b/_includes/v20.2/cdc/core-url.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +Because core changefeeds return results differently than other SQL statements, they require a dedicated database connection with specific settings around result buffering. In normal operation, CockroachDB improves performance by buffering results server-side before returning them to a client; however, result buffering is automatically turned off for core changefeeds. Core changefeeds also have different cancellation behavior than other queries: they can only be canceled by closing the underlying connection or issuing a [`CANCEL QUERY`](cancel-query.html) statement on a separate connection. Combined, these attributes of changefeeds mean that applications should explicitly create dedicated connections to consume changefeed data, instead of using a connection pool as most client drivers do by default. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/cdc/create-core-changefeed-avro.md b/_includes/v20.2/cdc/create-core-changefeed-avro.md new file mode 100644 index 00000000000..3846c0ffd34 --- /dev/null +++ b/_includes/v20.2/cdc/create-core-changefeed-avro.md @@ -0,0 +1,104 @@ +In this example, you'll set up a core changefeed for a single-node cluster that emits Avro records. CockroachDB's Avro binary encoding convention uses the [Confluent Schema Registry](https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html) to store Avro schemas. + +1. Use the [`cockroach start-single-node`](cockroach-start-single-node.html) command to start a single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node \ + --insecure \ + --listen-addr=localhost \ + --background + ~~~ + +2. Download and extract the [Confluent Open Source platform](https://www.confluent.io/download/). + +3. Move into the extracted `confluent-` directory and start Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent start + ~~~ + + Only `zookeeper`, `kafka`, and `schema-registry` are needed. To troubleshoot Confluent, see [their docs](https://docs.confluent.io/current/installation/installing_cp.html#zip-and-tar-archives). + +4. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --url="postgresql://root@127.0.0.1:26257?sslmode=disable" --format=csv + ~~~ + + {% include {{ page.version.version }}/cdc/core-url.md %} + + {% include {{ page.version.version }}/cdc/core-csv.md %} + +5. Enable the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING kv.rangefeed.enabled = true; + ~~~ + +6. Create table `bar`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bar (a INT PRIMARY KEY); + ~~~ + +7. Insert a row into the table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bar VALUES (0); + ~~~ + +8. Start the core changefeed: + + {% include copy-clipboard.html %} + ~~~ sql + > EXPERIMENTAL CHANGEFEED FOR bar WITH format = experimental_avro, confluent_schema_registry = 'http://localhost:8081'; + ~~~ + + ~~~ + table,key,value + bar,\000\000\000\000\001\002\000,\000\000\000\000\002\002\002\000 + ~~~ + +9. In a new terminal, add another row: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure -e "INSERT INTO bar VALUES (1)" + ~~~ + +10. Back in the terminal where the core changefeed is streaming, the output will appear: + + ~~~ + bar,\000\000\000\000\001\002\002,\000\000\000\000\002\002\002\002 + ~~~ + + Note that records may take a couple of seconds to display in the core changefeed. + +11. To stop streaming the changefeed, enter **CTRL+C** into the terminal where the changefeed is running. + +12. To stop `cockroach`, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure + ~~~ + +13. To stop Confluent, move into the extracted `confluent-` directory and stop Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent stop + ~~~ + + To stop all Confluent processes, use: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent destroy + ~~~ diff --git a/_includes/v20.2/cdc/create-core-changefeed.md b/_includes/v20.2/cdc/create-core-changefeed.md new file mode 100644 index 00000000000..0e9c876a00a --- /dev/null +++ b/_includes/v20.2/cdc/create-core-changefeed.md @@ -0,0 +1,80 @@ +In this example, you'll set up a core changefeed for a single-node cluster. + +1. In a terminal window, start `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --listen-addr=localhost \ + --background + ~~~ + +2. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --url="postgresql://root@127.0.0.1:26257?sslmode=disable" \ + --format=csv + ~~~ + + {% include {{ page.version.version }}/cdc/core-url.md %} + + {% include {{ page.version.version }}/cdc/core-csv.md %} + +3. Enable the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING kv.rangefeed.enabled = true; + ~~~ + +4. Create table `foo`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE foo (a INT PRIMARY KEY); + ~~~ + +5. Insert a row into the table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO foo VALUES (0); + ~~~ + +6. Start the core changefeed: + + {% include copy-clipboard.html %} + ~~~ sql + > EXPERIMENTAL CHANGEFEED FOR foo; + ~~~ + ~~~ + table,key,value + foo,[0],"{""after"": {""a"": 0}}" + ~~~ + +7. In a new terminal, add another row: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure -e "INSERT INTO foo VALUES (1)" + ~~~ + +8. Back in the terminal where the core changefeed is streaming, the following output has appeared: + + ~~~ + foo,[1],"{""after"": {""a"": 1}}" + ~~~ + + Note that records may take a couple of seconds to display in the core changefeed. + +9. To stop streaming the changefeed, enter **CTRL+C** into the terminal where the changefeed is running. + +10. To stop `cockroach`, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure + ~~~ diff --git a/_includes/v20.2/cdc/print-key.md b/_includes/v20.2/cdc/print-key.md new file mode 100644 index 00000000000..ab0b0924d30 --- /dev/null +++ b/_includes/v20.2/cdc/print-key.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +This example only prints the value. To print both the key and value of each message in the changefeed (e.g., to observe what happens with `DELETE`s), use the `--property print.key=true` flag. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/client-transaction-retry.md b/_includes/v20.2/client-transaction-retry.md new file mode 100644 index 00000000000..6a54534169e --- /dev/null +++ b/_includes/v20.2/client-transaction-retry.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +With the default `SERIALIZABLE` [isolation level](transactions.html#isolation-levels), CockroachDB may require the client to [retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a [generic retry function](transactions.html#client-side-intervention) that runs inside a transaction and retries it as needed. The code sample below shows how it is used. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/computed-columns/add-computed-column.md b/_includes/v20.2/computed-columns/add-computed-column.md new file mode 100644 index 00000000000..c670b1c7285 --- /dev/null +++ b/_includes/v20.2/computed-columns/add-computed-column.md @@ -0,0 +1,55 @@ +In this example, create a table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE x ( + a INT NULL, + b INT NULL AS (a * 2) STORED, + c INT NULL AS (a + 4) STORED, + FAMILY "primary" (a, b, rowid, c) + ); +~~~ + +Then, insert a row of data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO x VALUES (6); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM x; +~~~ + +~~~ ++---+----+----+ +| a | b | c | ++---+----+----+ +| 6 | 12 | 10 | ++---+----+----+ +(1 row) +~~~ + +Now add another computed column to the table: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE x ADD COLUMN d INT AS (a // 2) STORED; +~~~ + +The `d` column is added to the table and computed from the `a` column divided by 2. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM x; +~~~ + +~~~ ++---+----+----+---+ +| a | b | c | d | ++---+----+----+---+ +| 6 | 12 | 10 | 3 | ++---+----+----+---+ +(1 row) +~~~ diff --git a/_includes/v20.2/computed-columns/convert-computed-column.md b/_includes/v20.2/computed-columns/convert-computed-column.md new file mode 100644 index 00000000000..12fd6e7d418 --- /dev/null +++ b/_includes/v20.2/computed-columns/convert-computed-column.md @@ -0,0 +1,108 @@ +You can convert a stored, computed column into a regular column by using `ALTER TABLE`. + +In this example, create a simple table with a computed column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE office_dogs ( + id INT PRIMARY KEY, + first_name STRING, + last_name STRING, + full_name STRING AS (CONCAT(first_name, ' ', last_name)) STORED + ); +~~~ + +Then, insert a few rows of data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO office_dogs (id, first_name, last_name) VALUES + (1, 'Petee', 'Hirata'), + (2, 'Carl', 'Kimball'), + (3, 'Ernie', 'Narayan'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM office_dogs; +~~~ + +~~~ ++----+------------+-----------+---------------+ +| id | first_name | last_name | full_name | ++----+------------+-----------+---------------+ +| 1 | Petee | Hirata | Petee Hirata | +| 2 | Carl | Kimball | Carl Kimball | +| 3 | Ernie | Narayan | Ernie Narayan | ++----+------------+-----------+---------------+ +(3 rows) +~~~ + +The `full_name` column is computed from the `first_name` and `last_name` columns without the need to define a [view](views.html). You can view the column details with the [`SHOW COLUMNS`](show-columns.html) statement: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM office_dogs; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+------------------------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+------------------------------------+-------------+ +| id | INT | false | NULL | | {"primary"} | +| first_name | STRING | true | NULL | | {} | +| last_name | STRING | true | NULL | | {} | +| full_name | STRING | true | NULL | concat(first_name, ' ', last_name) | {} | ++-------------+-----------+-------------+----------------+------------------------------------+-------------+ +(4 rows) +~~~ + +Now, convert the computed column (`full_name`) to a regular column: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE office_dogs ALTER COLUMN full_name DROP STORED; +~~~ + +Check that the computed column was converted: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM office_dogs; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| id | INT | false | NULL | | {"primary"} | +| first_name | STRING | true | NULL | | {} | +| last_name | STRING | true | NULL | | {} | +| full_name | STRING | true | NULL | | {} | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +(4 rows) +~~~ + +The computed column is now a regular column and can be updated as such: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO office_dogs (id, first_name, last_name, full_name) VALUES (4, 'Lola', 'McDog', 'This is not computed'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM office_dogs; +~~~ + +~~~ ++----+------------+-----------+----------------------+ +| id | first_name | last_name | full_name | ++----+------------+-----------+----------------------+ +| 1 | Petee | Hirata | Petee Hirata | +| 2 | Carl | Kimball | Carl Kimball | +| 3 | Ernie | Narayan | Ernie Narayan | +| 4 | Lola | McDog | This is not computed | ++----+------------+-----------+----------------------+ +(4 rows) +~~~ diff --git a/_includes/v20.2/computed-columns/jsonb.md b/_includes/v20.2/computed-columns/jsonb.md new file mode 100644 index 00000000000..76a5b08ad8a --- /dev/null +++ b/_includes/v20.2/computed-columns/jsonb.md @@ -0,0 +1,35 @@ +In this example, create a table with a `JSONB` column and a computed column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE student_profiles ( + id STRING PRIMARY KEY AS (profile->>'id') STORED, + profile JSONB +); +~~~ + +Then, insert a few rows of data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO student_profiles (profile) VALUES + ('{"id": "d78236", "name": "Arthur Read", "age": "16", "school": "PVPHS", "credits": 120, "sports": "none"}'), + ('{"name": "Buster Bunny", "age": "15", "id": "f98112", "school": "THS", "credits": 67, "clubs": "MUN"}'), + ('{"name": "Ernie Narayan", "school" : "Brooklyn Tech", "id": "t63512", "sports": "Track and Field", "clubs": "Chess"}'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM student_profiles; +~~~ +~~~ ++--------+---------------------------------------------------------------------------------------------------------------------+ +| id | profile | ++--------+---------------------------------------------------------------------------------------------------------------------+ +| d78236 | {"age": "16", "credits": 120, "id": "d78236", "name": "Arthur Read", "school": "PVPHS", "sports": "none"} | +| f98112 | {"age": "15", "clubs": "MUN", "credits": 67, "id": "f98112", "name": "Buster Bunny", "school": "THS"} | +| t63512 | {"clubs": "Chess", "id": "t63512", "name": "Ernie Narayan", "school": "Brooklyn Tech", "sports": "Track and Field"} | ++--------+---------------------------------------------------------------------------------------------------------------------+ +~~~ + +The primary key `id` is computed as a field from the `profile` column. diff --git a/_includes/v20.2/computed-columns/partitioning.md b/_includes/v20.2/computed-columns/partitioning.md new file mode 100644 index 00000000000..926c45793b4 --- /dev/null +++ b/_includes/v20.2/computed-columns/partitioning.md @@ -0,0 +1,53 @@ +{{site.data.alerts.callout_info}}Partioning is an enterprise feature. To request and enable a trial or full enterprise license, see Enterprise Licensing.{{site.data.alerts.end}} + +In this example, create a table with geo-partitioning and a computed column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE user_locations ( + locality STRING AS (CASE + WHEN country IN ('ca', 'mx', 'us') THEN 'north_america' + WHEN country IN ('au', 'nz') THEN 'australia' + END) STORED, + id SERIAL, + name STRING, + country STRING, + PRIMARY KEY (locality, id)) + PARTITION BY LIST (locality) + (PARTITION north_america VALUES IN ('north_america'), + PARTITION australia VALUES IN ('australia')); +~~~ + +Then, insert a few rows of data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO user_locations (name, country) VALUES + ('Leonard McCoy', 'us'), + ('Uhura', 'nz'), + ('Spock', 'ca'), + ('James Kirk', 'us'), + ('Scotty', 'mx'), + ('Hikaru Sulu', 'us'), + ('Pavel Chekov', 'au'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM user_locations; +~~~ +~~~ ++---------------+--------------------+---------------+---------+ +| locality | id | name | country | ++---------------+--------------------+---------------+---------+ +| australia | 333153890100609025 | Uhura | nz | +| australia | 333153890100772865 | Pavel Chekov | au | +| north_america | 333153890100576257 | Leonard McCoy | us | +| north_america | 333153890100641793 | Spock | ca | +| north_america | 333153890100674561 | James Kirk | us | +| north_america | 333153890100707329 | Scotty | mx | +| north_america | 333153890100740097 | Hikaru Sulu | us | ++---------------+--------------------+---------------+---------+ +~~~ + +The `locality` column is computed from the `country` column. diff --git a/_includes/v20.2/computed-columns/secondary-index.md b/_includes/v20.2/computed-columns/secondary-index.md new file mode 100644 index 00000000000..e274db59d7e --- /dev/null +++ b/_includes/v20.2/computed-columns/secondary-index.md @@ -0,0 +1,63 @@ +In this example, create a table with a computed columns and an index on that column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE gymnastics ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + athlete STRING, + vault DECIMAL, + bars DECIMAL, + beam DECIMAL, + floor DECIMAL, + combined_score DECIMAL AS (vault + bars + beam + floor) STORED, + INDEX total (combined_score DESC) + ); +~~~ + +Then, insert a few rows a data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO gymnastics (athlete, vault, bars, beam, floor) VALUES + ('Simone Biles', 15.933, 14.800, 15.300, 15.800), + ('Gabby Douglas', 0, 15.766, 0, 0), + ('Laurie Hernandez', 15.100, 0, 15.233, 14.833), + ('Madison Kocian', 0, 15.933, 0, 0), + ('Aly Raisman', 15.833, 0, 15.000, 15.366); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM gymnastics; +~~~ +~~~ ++--------------------------------------+------------------+--------+--------+--------+--------+----------------+ +| id | athlete | vault | bars | beam | floor | combined_score | ++--------------------------------------+------------------+--------+--------+--------+--------+----------------+ +| 3fe11371-6a6a-49de-bbef-a8dd16560fac | Aly Raisman | 15.833 | 0 | 15.000 | 15.366 | 46.199 | +| 56055a70-b4c7-4522-909b-8f3674b705e5 | Madison Kocian | 0 | 15.933 | 0 | 0 | 15.933 | +| 69f73fd1-da34-48bf-aff8-71296ce4c2c7 | Gabby Douglas | 0 | 15.766 | 0 | 0 | 15.766 | +| 8a7b730b-668d-4845-8d25-48bda25114d6 | Laurie Hernandez | 15.100 | 0 | 15.233 | 14.833 | 45.166 | +| b2c5ca80-21c2-4853-9178-b96ce220ea4d | Simone Biles | 15.933 | 14.800 | 15.300 | 15.800 | 61.833 | ++--------------------------------------+------------------+--------+--------+--------+--------+----------------+ +~~~ + +Now, run a query using the secondary index: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT athlete, combined_score FROM gymnastics ORDER BY combined_score DESC; +~~~ +~~~ ++------------------+----------------+ +| athlete | combined_score | ++------------------+----------------+ +| Simone Biles | 61.833 | +| Aly Raisman | 46.199 | +| Laurie Hernandez | 45.166 | +| Madison Kocian | 15.933 | +| Gabby Douglas | 15.766 | ++------------------+----------------+ +~~~ + +The athlete with the highest combined score of 61.833 is Simone Biles. diff --git a/_includes/v20.2/computed-columns/simple.md b/_includes/v20.2/computed-columns/simple.md new file mode 100644 index 00000000000..49045fc6cb7 --- /dev/null +++ b/_includes/v20.2/computed-columns/simple.md @@ -0,0 +1,40 @@ +In this example, let's create a simple table with a computed column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + city STRING, + first_name STRING, + last_name STRING, + full_name STRING AS (CONCAT(first_name, ' ', last_name)) STORED, + address STRING, + credit_card STRING, + dl STRING UNIQUE CHECK (LENGTH(dl) < 8) +); +~~~ + +Then, insert a few rows of data: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users (first_name, last_name) VALUES + ('Lola', 'McDog'), + ('Carl', 'Kimball'), + ('Ernie', 'Narayan'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users; +~~~ +~~~ + id | city | first_name | last_name | full_name | address | credit_card | dl ++--------------------------------------+------+------------+-----------+---------------+---------+-------------+------+ + 5740da29-cc0c-47af-921c-b275d21d4c76 | NULL | Ernie | Narayan | Ernie Narayan | NULL | NULL | NULL + e7e0b748-9194-4d71-9343-cd65218848f0 | NULL | Lola | McDog | Lola McDog | NULL | NULL | NULL + f00e4715-8ca7-4d5a-8de5-ef1d5d8092f3 | NULL | Carl | Kimball | Carl Kimball | NULL | NULL | NULL +(3 rows) +~~~ + +The `full_name` column is computed from the `first_name` and `last_name` columns without the need to define a [view](views.html). diff --git a/_includes/v20.2/faq/auto-generate-unique-ids.html b/_includes/v20.2/faq/auto-generate-unique-ids.html new file mode 100644 index 00000000000..c1269995b2e --- /dev/null +++ b/_includes/v20.2/faq/auto-generate-unique-ids.html @@ -0,0 +1,107 @@ +To auto-generate unique row IDs, use the [`UUID`](uuid.html) column with the `gen_random_uuid()` [function](functions-and-operators.html#id-generation-functions) as the [default value](default-value.html): + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + city STRING NOT NULL, + name STRING NULL, + address STRING NULL, + credit_card STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + FAMILY "primary" (id, city, name, address, credit_card) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users (name, city) VALUES ('Petee', 'new york'), ('Eric', 'seattle'), ('Dan', 'seattle'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+-------+---------+-------------+ + cf8ee4e2-cd74-449a-b6e6-a0fb2017baa4 | new york | Petee | NULL | NULL + 2382564e-702f-42d9-a139-b6df535ae00a | seattle | Eric | NULL | NULL + 7d27e40b-263a-4891-b29b-d59135e55650 | seattle | Dan | NULL | NULL +(3 rows) +~~~ + +Alternatively, you can use the [`BYTES`](bytes.html) column with the `uuid_v4()` function as the default value instead: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users2 ( + id BYTES DEFAULT uuid_v4(), + city STRING NOT NULL, + name STRING NULL, + address STRING NULL, + credit_card STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + FAMILY "primary" (id, city, name, address, credit_card) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users2 (name, city) VALUES ('Anna', 'new york'), ('Jonah', 'seattle'), ('Terry', 'chicago'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users; +~~~ + +~~~ + id | city | name | address | credit_card ++------------------------------------------------+----------+-------+---------+-------------+ + 4\244\277\323/\261M\007\213\275*\0060\346\025z | chicago | Terry | NULL | NULL + \273*t=u.F\010\274f/}\313\332\373a | new york | Anna | NULL | NULL + \004\\\364nP\024L)\252\364\222r$\274O0 | seattle | Jonah | NULL | NULL +(3 rows) +~~~ + +In either case, generated IDs will be 128-bit, large enough for there to be virtually no chance of generating non-unique values. Also, once the table grows beyond a single key-value range (more than 512 MiB by default), new IDs will be scattered across all of the table's ranges and, therefore, likely across different nodes. This means that multiple nodes will share in the load. + +This approach has the disadvantage of creating a primary key that may not be useful in a query directly, which can require a join with another table or a secondary index. + +If it is important for generated IDs to be stored in the same key-value range, you can use an [integer type](int.html) with the `unique_rowid()` [function](functions-and-operators.html#id-generation-functions) as the default value, either explicitly or via the [`SERIAL` pseudo-type](serial.html): + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users3 ( + id INT DEFAULT unique_rowid(), + city STRING NOT NULL, + name STRING NULL, + address STRING NULL, + credit_card STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + FAMILY "primary" (id, city, name, address, credit_card) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users3 (name, city) VALUES ('Blake', 'chicago'), ('Hannah', 'seattle'), ('Bobby', 'seattle'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users3; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------+---------+--------+---------+-------------+ + 469048192112197633 | chicago | Blake | NULL | NULL + 469048192112263169 | seattle | Hannah | NULL | NULL + 469048192112295937 | seattle | Bobby | NULL | NULL +(3 rows) +~~~ + +Upon insert or upsert, the `unique_rowid()` function generates a default value from the timestamp and ID of the node executing the insert. Such time-ordered values are likely to be globally unique except in cases where a very large number of IDs (100,000+) are generated per node per second. Also, there can be gaps and the order is not completely guaranteed. diff --git a/_includes/v20.2/faq/clock-synchronization-effects.md b/_includes/v20.2/faq/clock-synchronization-effects.md new file mode 100644 index 00000000000..98e0d13888f --- /dev/null +++ b/_includes/v20.2/faq/clock-synchronization-effects.md @@ -0,0 +1,26 @@ +CockroachDB requires moderate levels of clock synchronization to preserve data consistency. For this reason, when a node detects that its clock is out of sync with at least half of the other nodes in the cluster by 80% of the maximum offset allowed, it spontaneously shuts down. This offset defaults to 500ms but can be changed via the [`--max-offset`](cockroach-start.html#flags-max-offset) flag when starting each node. + +While [serializable consistency](https://en.wikipedia.org/wiki/Serializability) is maintained regardless of clock skew, skew outside the configured clock offset bounds can result in violations of single-key linearizability between causally dependent transactions. It's therefore important to prevent clocks from drifting too far by running [NTP](http://www.ntp.org/) or other clock synchronization software on each node. + +The one rare case to note is when a node's clock suddenly jumps beyond the maximum offset before the node detects it. Although extremely unlikely, this could occur, for example, when running CockroachDB inside a VM and the VM hypervisor decides to migrate the VM to different hardware with a different time. In this case, there can be a small window of time between when the node's clock becomes unsynchronized and when the node spontaneously shuts down. During this window, it would be possible for a client to read stale data and write data derived from stale reads. To protect against this, we recommend using the `server.clock.forward_jump_check_enabled` and `server.clock.persist_upper_bound_interval` [cluster settings](cluster-settings.html). + +### Considerations + +When setting up clock synchronization: + +- All nodes in the cluster must be synced to the same time source, or to different sources that implement leap second smearing in the same way. For example, Google and Amazon have time sources that are compatible with each other (they implement [leap second smearing](https://developers.google.com/time/smear) in the same way), but are incompatible with the default NTP pool (which does not implement leap second smearing). +- For nodes running in AWS, we recommend [Amazon Time Sync Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html#configure-amazon-time-service). For nodes running in GCP, we recommend [Google's internal NTP service](https://cloud.google.com/compute/docs/instances/managing-instances#configure_ntp_for_your_instances). For nodes running elsewhere, we recommend [Google Public NTP](https://developers.google.com/time/). Note that the Google and Amazon time services can be mixed with each other, but they cannot be mixed with other time services (unless you have verified leap second behavior). Either all of your nodes should use the Google and Amazon services, or none of them should. +- If you do not want to use the Google or Amazon time sources, you can use [`chrony`](https://chrony.tuxfamily.org/index.html) and enable client-side leap smearing, unless the time source you're using already does server-side smearing. In most cases, we recommend the Google Public NTP time source because it handles smearing the leap second. If you use a different NTP time source that doesn't smear the leap second, you must configure client-side smearing manually and do so in the same way on each machine. +- Do not run more than one clock sync service on VMs where `cockroach` is running. + +### Tutorials + +For guidance on synchronizing clocks, see the tutorial for your deployment environment: + +Environment | Featured Approach +------------|--------------------- +[On-Premises](deploy-cockroachdb-on-premises.html#step-1-synchronize-clocks) | Use NTP with Google's external NTP service. +[AWS](deploy-cockroachdb-on-aws.html#step-3-synchronize-clocks) | Use the Amazon Time Sync Service. +[Azure](deploy-cockroachdb-on-microsoft-azure.html#step-3-synchronize-clocks) | Disable Hyper-V time synchronization and use NTP with Google's external NTP service. +[Digital Ocean](deploy-cockroachdb-on-digital-ocean.html#step-2-synchronize-clocks) | Use NTP with Google's external NTP service. +[GCE](deploy-cockroachdb-on-google-cloud-platform.html#step-3-synchronize-clocks) | Use NTP with Google's internal NTP service. diff --git a/_includes/v20.2/faq/clock-synchronization-monitoring.html b/_includes/v20.2/faq/clock-synchronization-monitoring.html new file mode 100644 index 00000000000..6db8b963acb --- /dev/null +++ b/_includes/v20.2/faq/clock-synchronization-monitoring.html @@ -0,0 +1,8 @@ +As explained in more detail [in our monitoring documentation](monitoring-and-alerting.html#prometheus-endpoint), each CockroachDB node exports a wide variety of metrics at `http://:/_status/vars` in the format used by the popular Prometheus timeseries database. Two of these metrics export how close each node's clock is to the clock of all other nodes: + +Metric | Definition +-------|----------- +`clock_offset_meannanos` | The mean difference between the node's clock and other nodes' clocks in nanoseconds +`clock_offset_stddevnanos` | The standard deviation of the difference between the node's clock and other nodes' clocks in nanoseconds + +As described in [the above answer](#what-happens-when-node-clocks-are-not-properly-synchronized), a node will kill itself if the mean offset of its clock from the other nodes' clocks exceeds 80% of the maximum offset allowed. It's recommended to monitor the `clock_offset_meannanos` metric and alert if it's approaching the 80% threshold of your cluster's configured max offset. diff --git a/_includes/v20.2/faq/differences-between-numberings.md b/_includes/v20.2/faq/differences-between-numberings.md new file mode 100644 index 00000000000..741ec4f8066 --- /dev/null +++ b/_includes/v20.2/faq/differences-between-numberings.md @@ -0,0 +1,11 @@ + +| Property | UUID generated with `uuid_v4()` | INT generated with `unique_rowid()` | Sequences | +|--------------------------------------|-----------------------------------------|-----------------------------------------------|--------------------------------| +| Size | 16 bytes | 8 bytes | 1 to 8 bytes | +| Ordering properties | Unordered | Highly time-ordered | Highly time-ordered | +| Performance cost at generation | Small, scalable | Small, scalable | Variable, can cause contention | +| Value distribution | Uniformly distributed (128 bits) | Contains time and space (node ID) components | Dense, small values | +| Data locality | Maximally distributed | Values generated close in time are co-located | Highly local | +| `INSERT` latency when used as key | Small, insensitive to concurrency | Small, but increases with concurrent INSERTs | Higher | +| `INSERT` throughput when used as key | Highest | Limited by max throughput on 1 node | Limited by max throughput on 1 node | +| Read throughput when used as key | Highest (maximal parallelism) | Limited | Limited | diff --git a/_includes/v20.2/faq/planned-maintenance.md b/_includes/v20.2/faq/planned-maintenance.md new file mode 100644 index 00000000000..e8a3562b602 --- /dev/null +++ b/_includes/v20.2/faq/planned-maintenance.md @@ -0,0 +1,22 @@ +By default, if a node stays offline for more than 5 minutes, the cluster will consider it dead and will rebalance its data to other nodes. Before temporarily stopping nodes for planned maintenance (e.g., upgrading system software), if you expect any nodes to be offline for longer than 5 minutes, you can prevent the cluster from unnecessarily rebalancing data off the nodes by increasing the `server.time_until_store_dead` [cluster setting](cluster-settings.html) to match the estimated maintenance window. + +For example, let's say you want to maintain a group of servers, and the nodes running on the servers may be offline for up to 15 minutes as a result. Before shutting down the nodes, you would change the `server.time_until_store_dead` cluster setting as follows: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING server.time_until_store_dead = '15m0s'; +~~~ + +After completing the maintenance work and [restarting the nodes](cockroach-start.html), you would then change the setting back to its default: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING server.time_until_store_dead = '5m0s'; +~~~ + +It's also important to ensure that load balancers do not send client traffic to a node about to be shut down, even if it will only be down for a few seconds. If you find that your load balancer's health check is not always recognizing a node as unready before the node shuts down, you can increase the `server.shutdown.drain_wait` setting, which tells the node to wait in an unready state for the specified duration. For example: + +{% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING server.shutdown.drain_wait = '10s'; + ~~~ diff --git a/_includes/v20.2/faq/sequential-numbers.md b/_includes/v20.2/faq/sequential-numbers.md new file mode 100644 index 00000000000..8a4794b9243 --- /dev/null +++ b/_includes/v20.2/faq/sequential-numbers.md @@ -0,0 +1,8 @@ +Sequential numbers can be generated in CockroachDB using the `unique_rowid()` built-in function or using [SQL sequences](create-sequence.html). However, note the following considerations: + +- Unless you need roughly-ordered numbers, we recommend using [`UUID`](uuid.html) values instead. See the [previous +FAQ](#how-do-i-auto-generate-unique-row-ids-in-cockroachdb) for details. +- [Sequences](create-sequence.html) produce **unique** values. However, not all values are guaranteed to be produced (e.g., when a transaction is canceled after it consumes a value) and the values may be slightly reordered (e.g., when a transaction that +consumes a lower sequence number commits after a transaction that consumes a higher number). +- For maximum performance, avoid using sequences or `unique_rowid()` to generate row IDs or indexed columns. Values generated in these ways are logically close to each other and can cause contention on few data ranges during inserts. Instead, prefer [`UUID`](uuid.html) identifiers. +- {% include {{page.version.version}}/performance/use-hash-sharded-indexes.md %} diff --git a/_includes/v20.2/faq/sequential-transactions.md b/_includes/v20.2/faq/sequential-transactions.md new file mode 100644 index 00000000000..684f2ce5d2a --- /dev/null +++ b/_includes/v20.2/faq/sequential-transactions.md @@ -0,0 +1,19 @@ +Most use cases that ask for a strong time-based write ordering can be solved with other, more distribution-friendly +solutions instead. For example, CockroachDB's [time travel queries (`AS OF SYSTEM +TIME`)](https://www.cockroachlabs.com/blog/time-travel-queries-select-witty_subtitle-the_future/) support the following: + +- Paginating through all the changes to a table or dataset +- Determining the order of changes to data over time +- Determining the state of data at some point in the past +- Determining the changes to data between two points of time + +Consider also that the values generated by `unique_rowid()`, described in the previous FAQ entries, also provide an approximate time ordering. + +However, if your application absolutely requires strong time-based write ordering, it is possible to create a strictly monotonic counter in CockroachDB that increases over time as follows: + +- Initially: `CREATE TABLE cnt(val INT PRIMARY KEY); INSERT INTO cnt(val) VALUES(1);` +- In each transaction: `INSERT INTO cnt(val) SELECT max(val)+1 FROM cnt RETURNING val;` + +This will cause [`INSERT`](insert.html) transactions to conflict with each other and effectively force the transactions to commit one at a time throughout the cluster, which in turn guarantees the values generated in this way are strictly increasing over time without gaps. The caveat is that performance is severely limited as a result. + +If you find yourself interested in this problem, please [contact us](support-resources.html) and describe your situation. We would be glad to help you find alternative solutions and possibly extend CockroachDB to better match your needs. diff --git a/_includes/v20.2/faq/simulate-key-value-store.html b/_includes/v20.2/faq/simulate-key-value-store.html new file mode 100644 index 00000000000..4772fa5358c --- /dev/null +++ b/_includes/v20.2/faq/simulate-key-value-store.html @@ -0,0 +1,13 @@ +CockroachDB is a distributed SQL database built on a transactional and strongly-consistent key-value store. Although it is not possible to access the key-value store directly, you can mirror direct access using a "simple" table of two columns, with one set as the primary key: + +~~~ sql +> CREATE TABLE kv (k INT PRIMARY KEY, v BYTES); +~~~ + +When such a "simple" table has no indexes or foreign keys, [`INSERT`](insert.html)/[`UPSERT`](upsert.html)/[`UPDATE`](update.html)/[`DELETE`](delete.html) statements translate to key-value operations with minimal overhead (single digit percent slowdowns). For example, the following `UPSERT` to add or replace a row in the table would translate into a single key-value Put operation: + +~~~ sql +> UPSERT INTO kv VALUES (1, b'hello') +~~~ + +This SQL table approach also offers you a well-defined query language, a known transaction model, and the flexibility to add more columns to the table if the need arises. diff --git a/_includes/v20.2/faq/sql-query-logging.md b/_includes/v20.2/faq/sql-query-logging.md new file mode 100644 index 00000000000..4e3a8b27780 --- /dev/null +++ b/_includes/v20.2/faq/sql-query-logging.md @@ -0,0 +1,143 @@ +There are several ways to log SQL queries. The type of logging you use will depend on your requirements. + +- For per-table audit logs, turn on [SQL audit logs](#sql-audit-logs). +- For system troubleshooting and performance optimization, turn on [cluster-wide execution logs](#cluster-wide-execution-logs) and [slow query logs](#slow-query-logs). +- For connection troubleshooting, turn on [authentication logs](#authentication-logs). +- For local testing, turn on [per-node execution logs](#per-node-execution-logs). + +### SQL audit logs + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +SQL audit logging is useful if you want to log all queries that are run against specific tables. + +- For a tutorial, see [SQL Audit Logging](sql-audit-logging.html). +- For SQL reference documentation, see [`ALTER TABLE ... EXPERIMENTAL_AUDIT`](experimental-audit.html). +- Note that SQL audit logs perform one disk I/O per event and will impact performance. + +### Cluster-wide execution logs + +For production clusters, the best way to log all queries is to turn on the [cluster-wide setting](cluster-settings.html) `sql.trace.log_statement_execute`: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.trace.log_statement_execute = true; +~~~ + +With this setting on, each node of the cluster writes all SQL queries it executes to a secondary `cockroach-sql-exec` log file. Use the symlink `cockroach-sql-exec.log` to open the most recent log. When you no longer need to log queries, you can turn the setting back off: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.trace.log_statement_execute = false; +~~~ + +Log files are written to CockroachDB's standard [log directory](debug-and-error-logs.html#write-to-file). + +### Slow query logs + + Another useful [cluster setting](cluster-settings.html) is `sql.log.slow_query.latency_threshold`, which is used to log only queries whose service latency exceeds a specified threshold value (e.g., 100 milliseconds): + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.log.slow_query.latency_threshold = '100ms'; +~~~ + +Each node that serves as a gateway will then record slow SQL queries to a `cockroach-sql-slow` log file. Use the symlink `cockroach-sql-slow.log` to open the most recent log. For more details on logging slow queries, see [Using the slow query log](query-behavior-troubleshooting.html#using-the-slow-query-log). + +Log files are written to CockroachDB's standard [log directory](debug-and-error-logs.html#write-to-file). + +### Authentication logs + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +SQL client connections can be logged by turning on the `server.auth_log.sql_connections.enabled` [cluster setting](cluster-settings.html): + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING server.auth_log.sql_connections.enabled = true; +~~~ + +This will log connection established and connection terminated events to a `cockroach-auth` log file. Use the symlink `cockroach-auth.log` to open the most recent log. + +{{site.data.alerts.callout_info}} +In addition to SQL sessions, connection events can include SQL-based liveness probe attempts, as well as attempts to use the [PostgreSQL cancel protocol](https://www.postgresql.org/docs/current/protocol-flow.html#id-1.10.5.7.9). +{{site.data.alerts.end}} + +This example log shows both types of connection events over a `hostssl` (TLS certificate over TCP) connection: + +~~~ +I200219 05:08:43.083907 5235 sql/pgwire/server.go:445 [n1,client=[::1]:34588] 22 received connection +I200219 05:08:44.171384 5235 sql/pgwire/server.go:453 [n1,client=[::1]:34588,hostssl] 26 disconnected; duration: 1.087489893s +~~~ + +Along with the above, SQL client authenticated sessions can be logged by turning on the `server.auth_log.sql_sessions.enabled` [cluster setting](cluster-settings.html): + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING server.auth_log.sql_sessions.enabled = true; +~~~ + +This logs authentication method selection, authentication method application, authentication method result, and session termination events to the `cockroach-auth` log file. Use the symlink `cockroach-auth.log` to open the most recent log. + +This example log shows authentication success over a `hostssl` (TLS certificate over TCP) connection: + +~~~ +I200219 05:08:43.089501 5149 sql/pgwire/auth.go:327 [n1,client=[::1]:34588,hostssl,user=root] 23 connection matches HBA rule: +# TYPE DATABASE USER ADDRESS METHOD OPTIONS +host all root all cert-password +I200219 05:08:43.091045 5149 sql/pgwire/auth.go:327 [n1,client=[::1]:34588,hostssl,user=root] 24 authentication succeeded +I200219 05:08:44.169684 5235 sql/pgwire/conn.go:216 [n1,client=[::1]:34588,hostssl,user=root] 25 session terminated; duration: 1.080240961s +~~~ + +This example log shows authentication failure log over a `local` (password over Unix socket) connection: + +~~~ +I200219 05:02:18.148961 1037 sql/pgwire/auth.go:327 [n1,client,local,user=root] 17 connection matches HBA rule: +# TYPE DATABASE USER ADDRESS METHOD OPTIONS +local all all password +I200219 05:02:18.151644 1037 sql/pgwire/auth.go:327 [n1,client,local,user=root] 18 user has no password defined +I200219 05:02:18.152863 1037 sql/pgwire/auth.go:327 [n1,client,local,user=root] 19 authentication failed: password authentication failed for user root +I200219 05:02:18.154168 1036 sql/pgwire/conn.go:216 [n1,client,local,user=root] 20 session terminated; duration: 5.261538ms +~~~ + +For complete logging of client connections, we recommend enabling both `server.auth_log.sql_connections.enabled` and `server.auth_log.sql_sessions.enabled`. Note that both logs perform one disk I/O per event and will impact performance. + +For more details on authentication and certificates, see [Authentication](authentication.html). + +Log files are written to CockroachDB's standard [log directory](debug-and-error-logs.html#write-to-file). + +### Per-node execution logs + +Alternatively, if you are testing CockroachDB locally and want to log queries executed just by a specific node, you can either pass a CLI flag at node startup, or execute a SQL function on a running node. + +Using the CLI to start a new node, pass the `--vmodule` flag to the [`cockroach start`](cockroach-start.html) command. For example, to start a single node locally and log all client-generated SQL queries it executes, you'd run: + +~~~ shell +$ cockroach start --insecure --listen-addr=localhost --vmodule=exec_log=2 --join= +~~~ + +{{site.data.alerts.callout_success}} +To log CockroachDB-generated SQL queries as well, use `--vmodule=exec_log=3`. +{{site.data.alerts.end}} + +From the SQL prompt on a running node, execute the `crdb_internal.set_vmodule()` [function](functions-and-operators.html): + +{% include copy-clipboard.html %} +~~~ sql +> SELECT crdb_internal.set_vmodule('exec_log=2'); +~~~ + +This will result in the following output: + +~~~ + crdb_internal.set_vmodule ++---------------------------+ + 0 +(1 row) +~~~ + +Once the logging is enabled, all client-generated SQL queries executed by the node will be written to the primary [CockroachDB log file](debug-and-error-logs.html) as follows: + +~~~ +I180402 19:12:28.112957 394661 sql/exec_log.go:173 [n1,client=127.0.0.1:50155,user=root] exec "psql" {} "SELECT version()" {} 0.795 1 "" +~~~ diff --git a/_includes/v20.2/faq/when-to-interleave-tables.html b/_includes/v20.2/faq/when-to-interleave-tables.html new file mode 100644 index 00000000000..a65196ad693 --- /dev/null +++ b/_includes/v20.2/faq/when-to-interleave-tables.html @@ -0,0 +1,5 @@ +You're most likely to benefit from interleaved tables when: + + - Your tables form a [hierarchy](interleave-in-parent.html#interleaved-hierarchy) + - Queries maximize the [benefits of interleaving](interleave-in-parent.html#benefits) + - Queries do not suffer too greatly from interleaving's [tradeoffs](interleave-in-parent.html#tradeoffs) diff --git a/_includes/v20.2/json/json-sample.go b/_includes/v20.2/json/json-sample.go new file mode 100644 index 00000000000..75b15e95baf --- /dev/null +++ b/_includes/v20.2/json/json-sample.go @@ -0,0 +1,79 @@ +package main + +import ( + "database/sql" + "fmt" + "io/ioutil" + "net/http" + "time" + + _ "github.com/lib/pq" +) + +func main() { + db, err := sql.Open("postgres", "user=maxroach dbname=jsonb_test sslmode=disable port=26257") + if err != nil { + panic(err) + } + + // The Reddit API wants us to tell it where to start from. The first request + // we just say "null" to say "from the start", subsequent requests will use + // the value received from the last call. + after := "null" + + for i := 0; i < 300; i++ { + after, err = makeReq(db, after) + if err != nil { + panic(err) + } + // Reddit limits to 30 requests per minute, so don't do any more than that. + time.Sleep(2 * time.Second) + } +} + +func makeReq(db *sql.DB, after string) (string, error) { + // First, make a request to reddit using the appropriate "after" string. + client := &http.Client{} + req, err := http.NewRequest("GET", fmt.Sprintf("https://www.reddit.com/r/programming.json?after=%s", after), nil) + + req.Header.Add("User-Agent", `Go`) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + + res, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + // We've gotten back our JSON from reddit, we can use a couple SQL tricks to + // accomplish multiple things at once. + // The JSON reddit returns looks like this: + // { + // "data": { + // "children": [ ... ] + // }, + // "after": ... + // } + // We structure our query so that we extract the `children` field, and then + // expand that and insert each individual element into the database as a + // separate row. We then return the "after" field so we know how to make the + // next request. + r, err := db.Query(` + INSERT INTO jsonb_test.programming (posts) + SELECT json_array_elements($1->'data'->'children') + RETURNING $1->'data'->'after'`, + string(res)) + if err != nil { + return "", err + } + + // Since we did a RETURNING, we need to grab the result of our query. + r.Next() + var newAfter string + r.Scan(&newAfter) + + return newAfter, nil +} diff --git a/_includes/v20.2/json/json-sample.py b/_includes/v20.2/json/json-sample.py new file mode 100644 index 00000000000..64ab9dad0d0 --- /dev/null +++ b/_includes/v20.2/json/json-sample.py @@ -0,0 +1,44 @@ +import json +import psycopg2 +import requests +import time + +conn = psycopg2.connect(database="jsonb_test", user="maxroach", host="localhost", port=26257) +conn.set_session(autocommit=True) +cur = conn.cursor() + +# The Reddit API wants us to tell it where to start from. The first request +# we just say "null" to say "from the start"; subsequent requests will use +# the value received from the last call. +url = "https://www.reddit.com/r/programming.json" +after = {"after": "null"} + +for n in range(300): + # First, make a request to reddit using the appropriate "after" string. + req = requests.get(url, params=after, headers={"User-Agent": "Python"}) + + # Decode the JSON and set "after" for the next request. + resp = req.json() + after = {"after": str(resp['data']['after'])} + + # Convert the JSON to a string to send to the database. + data = json.dumps(resp) + + # The JSON reddit returns looks like this: + # { + # "data": { + # "children": [ ... ] + # }, + # "after": ... + # } + # We structure our query so that we extract the `children` field, and then + # expand that and insert each individual element into the database as a + # separate row. + cur.execute("""INSERT INTO jsonb_test.programming (posts) + SELECT json_array_elements(%s->'data'->'children')""", (data,)) + + # Reddit limits to 30 requests per minute, so don't do any more than that. + time.sleep(2) + +cur.close() +conn.close() diff --git a/_includes/v20.2/known-limitations/adding-stores-to-node.md b/_includes/v20.2/known-limitations/adding-stores-to-node.md new file mode 100644 index 00000000000..206d98718a3 --- /dev/null +++ b/_includes/v20.2/known-limitations/adding-stores-to-node.md @@ -0,0 +1,5 @@ +After a node has initially joined a cluster, it is not possible to add additional [stores](cockroach-start.html#store) to the node. Stopping the node and restarting it with additional stores causes the node to not reconnect to the cluster. + +To work around this limitation, [decommission the node](remove-nodes.html), remove its data directory, and then run [`cockroach start`](cockroach-start.html) to join the cluster again as a new node. + +[Tracking GitHub Issue](https://github.com/cockroachdb/cockroach/issues/39415) diff --git a/_includes/v20.2/known-limitations/cdc.md b/_includes/v20.2/known-limitations/cdc.md new file mode 100644 index 00000000000..80f482a1cd0 --- /dev/null +++ b/_includes/v20.2/known-limitations/cdc.md @@ -0,0 +1,10 @@ +- Changefeeds only work on tables with a single [column family](column-families.html) (which is the default for new tables). +- Changefeeds do not share internal buffers, so each running changefeed will increase total memory usage. To watch multiple tables, we recommend creating a changefeed with a comma-separated list of tables. +- Many DDL queries (including [`TRUNCATE`](truncate.html) and [`DROP TABLE`](drop-table.html)) will cause errors on a changefeed watching the affected tables. You will need to [start a new changefeed](create-changefeed.html#start-a-new-changefeed-where-another-ended). +- Changefeeds cannot be [backed up](backup.html) or [restored](restore.html). +- Partial or intermittent sink unavailability may impact changefeed stability; however, [ordering guarantees](change-data-capture.html#ordering-guarantees) will still hold for as long as a changefeed [remains active](change-data-capture.html#monitor-a-changefeed). +- Changefeeds cannot be altered. To alter, cancel the changefeed and [create a new one with updated settings from where it left off](create-changefeed.html#start-a-new-changefeed-where-another-ended). +- Additional target options will be added, including partitions and ranges of primary key rows. +- There is an open correctness issue with changefeeds connected to cloud storage sinks where new row information will display with a lower timestamp than what has already been emitted, which violates our [ordering guarantees](change-data-capture.html#ordering-guarantees). +- Changefeeds do not pick up data ingested with the [`IMPORT INTO`](import-into.html) statement. +- Using a [cloud storage sink](create-changefeed.html#cloud-storage-sink) only works with `JSON` and emits [newline-delimited json](http://ndjson.org) files. diff --git a/_includes/v20.2/known-limitations/correlated-ctes.md b/_includes/v20.2/known-limitations/correlated-ctes.md new file mode 100644 index 00000000000..165c052816b --- /dev/null +++ b/_includes/v20.2/known-limitations/correlated-ctes.md @@ -0,0 +1,24 @@ +### Correlated common table expressions + +CockroachDB does not support correlated common table expressions. This means that a CTE cannot refer to a variable defined outside the scope of that CTE. + +For example, the following query returns an error: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users + WHERE id = + (WITH rides_home AS + (SELECT revenue FROM rides + WHERE end_address = address) + SELECT rider_id FROM rides_home); +~~~ + +~~~ +ERROR: CTEs may not be correlated +SQLSTATE: 0A000 +~~~ + +This query returns an error because the `WITH rides_home` clause references a column (`address`) returned by the `SELECT` statement at the top level of the query, outside the `rides_home` CTE definition. + +For details, see the tracking issue: [cockroachdb/cockroach#42540](https://github.com/cockroachdb/cockroach/issues/42540). \ No newline at end of file diff --git a/_includes/v20.2/known-limitations/dropping-renaming-during-upgrade.md b/_includes/v20.2/known-limitations/dropping-renaming-during-upgrade.md new file mode 100644 index 00000000000..38f7f9ddd87 --- /dev/null +++ b/_includes/v20.2/known-limitations/dropping-renaming-during-upgrade.md @@ -0,0 +1,10 @@ +When upgrading from v20.1.x to v20.2.0, as soon as any node of the cluster has run v20.2.0, it is important to avoid dropping, renaming, or truncating tables, views, sequences, or databases on the v20.1 nodes. This is true even in cases where nodes were upgraded to v20.2.0 and then rolled back to v20.1. + +In this case, avoid running the following operations against v20.1 nodes: + +- [`DROP TABLE`](drop-table.html), [`TRUNCATE TABLE`](truncate.html), [`RENAME TABLE`](rename-table.html) +- [`DROP VIEW`](drop-view.html) +- [`DROP SEQUENCE`](drop-sequence.html), [`RENAME SEQUENCE`](rename-sequence.html) +- [`DROP DATABASE`](drop-database.html), [`RENAME DATABASE`](rename-database.html) + +Running any of these operations against v19.2 nodes will result in inconsistency between two internal tables, `system.namespace` and `system.namespace2`. This inconsistency will prevent you from being able to recreate the dropped or renamed objects; the returned error will be `ERROR: relation already exists`. In the case of a dropped or renamed database, [`SHOW DATABASES`](show-databases.html) will also return an error: `ERROR: internal error: "" is not a database`. diff --git a/_includes/v20.2/known-limitations/dump-table-with-collations.md b/_includes/v20.2/known-limitations/dump-table-with-collations.md new file mode 100644 index 00000000000..50c700b0e1b --- /dev/null +++ b/_includes/v20.2/known-limitations/dump-table-with-collations.md @@ -0,0 +1,55 @@ +When using [`cockroach dump`](cockroach-dump.html) to dump the data of a table containing [collations](collate.html), the resulting `INSERT`s do not include the relevant collation clauses. For example: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start-single-node --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE de_names (name STRING COLLATE de PRIMARY KEY); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO de_names VALUES + ('Backhaus' COLLATE de), + ('Bär' COLLATE de), + ('Baz' COLLATE de) + ; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> q +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump defaultdb de_names --insecure > dump.sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat dump.sql +~~~ + +~~~ +CREATE TABLE de_names ( + name STRING COLLATE de NOT NULL, + CONSTRAINT "primary" PRIMARY KEY (name ASC), + FAMILY "primary" (name) +); + +INSERT INTO de_names (name) VALUES + ('Backhaus'), + (e'B\u00E4r'), + ('Baz'); +~~~ + +[Tracking GitHub Issue](https://github.com/cockroachdb/cockroach/issues/48278) diff --git a/_includes/v20.2/known-limitations/dump-table-with-no-columns.md b/_includes/v20.2/known-limitations/dump-table-with-no-columns.md new file mode 100644 index 00000000000..9dc903636c5 --- /dev/null +++ b/_includes/v20.2/known-limitations/dump-table-with-no-columns.md @@ -0,0 +1 @@ +It is not currently possible to use [`cockroach dump`](cockroach-dump.html) to dump the schema and data of a table with no user-defined columns. See [#35462](https://github.com/cockroachdb/cockroach/issues/35462) for more details. diff --git a/_includes/v20.2/known-limitations/import-high-disk-contention.md b/_includes/v20.2/known-limitations/import-high-disk-contention.md new file mode 100644 index 00000000000..48b9c63acf2 --- /dev/null +++ b/_includes/v20.2/known-limitations/import-high-disk-contention.md @@ -0,0 +1,6 @@ +[`IMPORT`](import.html) can sometimes fail with a "context canceled" error, or can restart itself many times without ever finishing. If this is happening, it is likely due to a high amount of disk contention. This can be mitigated by setting the `kv.bulk_io_write.max_rate` [cluster setting](cluster-settings.html) to a value below your max disk write speed. For example, to set it to 10MB/s, execute: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING kv.bulk_io_write.max_rate = '10MB'; +~~~ diff --git a/_includes/v20.2/known-limitations/import-interleaved-table.md b/_includes/v20.2/known-limitations/import-interleaved-table.md new file mode 100644 index 00000000000..1cde934b496 --- /dev/null +++ b/_includes/v20.2/known-limitations/import-interleaved-table.md @@ -0,0 +1 @@ +After using [`cockroach dump`](cockroach-dump.html) to dump the schema and data of an interleaved table, the output must be edited before it can be imported via [`IMPORT`](import.html). See [#35462](https://github.com/cockroachdb/cockroach/issues/35462) for the workaround and more details. diff --git a/_includes/v20.2/known-limitations/node-map.md b/_includes/v20.2/known-limitations/node-map.md new file mode 100644 index 00000000000..df9ef58486e --- /dev/null +++ b/_includes/v20.2/known-limitations/node-map.md @@ -0,0 +1,8 @@ +You cannot assign latitude/longitude coordinates to localities if the components of your localities have the same name. For example, consider the following partial configuration: + +| Node | Region | Datacenter | +| ------ | ------ | ------ | +| Node1 | us-east | datacenter-1 | +| Node2 | us-west | datacenter-1 | + +In this case, if you try to set the latitude/longitude coordinates to the datacenter level of the localities, you will get the "primary key exists" error and the Node Map will not be displayed. You can, however, set the latitude/longitude coordinates to the region components of the localities, and the Node Map will be displayed. diff --git a/_includes/v20.2/known-limitations/partitioning-with-placeholders.md b/_includes/v20.2/known-limitations/partitioning-with-placeholders.md new file mode 100644 index 00000000000..b3c3345200d --- /dev/null +++ b/_includes/v20.2/known-limitations/partitioning-with-placeholders.md @@ -0,0 +1 @@ +When defining a [table partition](partitioning.html), either during table creation or table alteration, it is not possible to use placeholders in the `PARTITION BY` clause. diff --git a/_includes/v20.2/known-limitations/schema-change-ddl-inside-multi-statement-transactions.md b/_includes/v20.2/known-limitations/schema-change-ddl-inside-multi-statement-transactions.md new file mode 100644 index 00000000000..b7d947bb4c9 --- /dev/null +++ b/_includes/v20.2/known-limitations/schema-change-ddl-inside-multi-statement-transactions.md @@ -0,0 +1,64 @@ +Schema change [DDL](https://en.wikipedia.org/wiki/Data_definition_language#ALTER_statement) statements that run inside a multi-statement transaction with non-DDL statements can fail at [`COMMIT`](commit-transaction.html) time, even if other statements in the transaction succeed. This leaves such transactions in a "partially committed, partially aborted" state that may require manual intervention to determine whether the DDL statements succeeded. + +If such a failure occurs, CockroachDB will emit a new CockroachDB-specific error code, `XXA00`, and the following error message: + +``` +transaction committed but schema change aborted with error: +HINT: Some of the non-DDL statements may have committed successfully, but some of the DDL statement(s) failed. +Manual inspection may be required to determine the actual state of the database. +``` + +{{site.data.alerts.callout_info}} +This limitation exists in versions of CockroachDB prior to 19.2. In these older versions, CockroachDB returned the Postgres error code `40003`, `"statement completion unknown"`. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_danger}} +If you must execute schema change DDL statements inside a multi-statement transaction, we **strongly recommend** checking for this error code and handling it appropriately every time you execute such transactions. +{{site.data.alerts.end}} + +This error will occur in various scenarios, including but not limited to: + +- Creating a unique index fails because values aren't unique. +- The evaluation of a computed value fails. +- Adding a constraint (or a column with a constraint) fails because the constraint is violated for the default/computed values in the column. + +To see an example of this error, start by creating the following table. + +{% include copy-clipboard.html %} +~~~ sql +CREATE TABLE T(x INT); +INSERT INTO T(x) VALUES (1), (2), (3); +~~~ + +Then, enter the following multi-statement transaction, which will trigger the error. + +{% include copy-clipboard.html %} +~~~ sql +BEGIN; +ALTER TABLE t ADD CONSTRAINT unique_x UNIQUE(x); +INSERT INTO T(x) VALUES (3); +COMMIT; +~~~ + +~~~ +pq: transaction committed but schema change aborted with error: (23505): duplicate key value (x)=(3) violates unique constraint "unique_x" +HINT: Some of the non-DDL statements may have committed successfully, but some of the DDL statement(s) failed. +Manual inspection may be required to determine the actual state of the database. +~~~ + +In this example, the [`INSERT`](insert.html) statement committed, but the [`ALTER TABLE`](alter-table.html) statement adding a [`UNIQUE` constraint](unique.html) failed. We can verify this by looking at the data in table `t` and seeing that the additional non-unique value `3` was successfully inserted. + +{% include copy-clipboard.html %} +~~~ sql +SELECT * FROM t; +~~~ + +~~~ + x ++---+ + 1 + 2 + 3 + 3 +(4 rows) +~~~ diff --git a/_includes/v20.2/known-limitations/schema-changes-between-prepared-statements.md b/_includes/v20.2/known-limitations/schema-changes-between-prepared-statements.md new file mode 100644 index 00000000000..c739262b4b8 --- /dev/null +++ b/_includes/v20.2/known-limitations/schema-changes-between-prepared-statements.md @@ -0,0 +1,42 @@ +When the schema of a table targeted by a prepared statement changes before the prepared statement is executed, CockroachDB allows the prepared statement to return results based on the changed table schema, for example: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users (id INT PRIMARY KEY); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> PREPARE prep1 AS SELECT * FROM users; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD COLUMN name STRING; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users VALUES (1, 'Max Roach'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXECUTE prep1; +~~~ + +~~~ ++----+-----------+ +| id | name | ++----+-----------+ +| 1 | Max Roach | ++----+-----------+ +(1 row) +~~~ + +It's therefore recommended to **not** use `SELECT *` in queries that will be repeated, via prepared statements or otherwise. + +Also, a prepared [`INSERT`](insert.html), [`UPSERT`](upsert.html), or [`DELETE`](delete.html) statement acts inconsistently when the schema of the table being written to is changed before the prepared statement is executed: + +- If the number of columns has increased, the prepared statement returns an error but nonetheless writes the data. +- If the number of columns remains the same but the types have changed, the prepared statement writes the data and does not return an error. diff --git a/_includes/v20.2/known-limitations/schema-changes-within-transactions.md b/_includes/v20.2/known-limitations/schema-changes-within-transactions.md new file mode 100644 index 00000000000..89ece41a023 --- /dev/null +++ b/_includes/v20.2/known-limitations/schema-changes-within-transactions.md @@ -0,0 +1,12 @@ +Within a single [transaction](transactions.html): + +- DDL statements cannot be mixed with DML statements. As a workaround, you can split the statements into separate transactions. For more details, [see examples of unsupported statements](online-schema-changes.html#examples-of-statements-that-fail). +- As of version v2.1, you can run schema changes inside the same transaction as a [`CREATE TABLE`](create-table.html) statement. For more information, [see this example](online-schema-changes.html#run-schema-changes-inside-a-transaction-with-create-table). +- A `CREATE TABLE` statement containing [`FOREIGN KEY`](foreign-key.html) or [`INTERLEAVE`](interleave-in-parent.html) clauses cannot be followed by statements that reference the new table. +- A table cannot be dropped and then recreated with the same name. This is not possible within a single transaction because `DROP TABLE` does not immediately drop the name of the table. As a workaround, split the [`DROP TABLE`](drop-table.html) and [`CREATE TABLE`](create-table.html) statements into separate transactions. +- [Schema change DDL statements inside a multi-statement transaction can fail while other statements succeed](#schema-change-ddl-statements-inside-a-multi-statement-transaction-can-fail-while-other-statements-succeed). +- As of v19.1, some schema changes can be used in combination in a single `ALTER TABLE` statement. For a list of commands that can be combined, see [`ALTER TABLE`](alter-table.html). For a demonstration, see [Add and rename columns atomically](rename-column.html#add-and-rename-columns-atomically). + +{{site.data.alerts.callout_info}} +If a schema change within a transaction fails, manual intervention may be needed to determine which has failed. After determining which schema change(s) failed, you can then retry the schema changes. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/known-limitations/system-range-replication.md b/_includes/v20.2/known-limitations/system-range-replication.md new file mode 100644 index 00000000000..1d649d04834 --- /dev/null +++ b/_includes/v20.2/known-limitations/system-range-replication.md @@ -0,0 +1 @@ +Changes to the [`default` cluster-wide replication zone](configure-replication-zones.html#edit-the-default-replication-zone) are automatically applied to existing replication zones, including pre-configured zones for important system ranges that must remain available for the cluster as a whole to remain available. The zones for these system ranges have an initial replication factor of 5 to make them more resilient to node failure. However, if you increase the `default` zone's replication factor above 5, consider [increasing the replication factor for important system ranges](configure-replication-zones.html#create-a-replication-zone-for-a-system-range) as well. diff --git a/_includes/v20.2/metric-names.md b/_includes/v20.2/metric-names.md new file mode 100644 index 00000000000..80098b223b9 --- /dev/null +++ b/_includes/v20.2/metric-names.md @@ -0,0 +1,246 @@ +Name | Help +-----|----- +`addsstable.applications` | Number of SSTable ingestions applied (i.e., applied by Replicas) +`addsstable.copies` | Number of SSTable ingestions that required copying files during application +`addsstable.proposals` | Number of SSTable ingestions proposed (i.e., sent to Raft by lease holders) +`build.timestamp` | Build information +`capacity.available` | Available storage capacity +`capacity.reserved` | Capacity reserved for snapshots +`capacity.used` | Used storage capacity +`capacity` | Total storage capacity +`clock-offset.meannanos` | Mean clock offset with other nodes in nanoseconds +`clock-offset.stddevnanos` | Std dev clock offset with other nodes in nanoseconds +`compactor.compactingnanos` | Number of nanoseconds spent compacting ranges +`compactor.compactions.failure` | Number of failed compaction requests sent to the storage engine +`compactor.compactions.success` | Number of successful compaction requests sent to the storage engine +`compactor.suggestionbytes.compacted` | Number of logical bytes compacted from suggested compactions +`compactor.suggestionbytes.queued` | Number of logical bytes in suggested compactions in the queue +`compactor.suggestionbytes.skipped` | Number of logical bytes in suggested compactions which were not compacted +`distsender.batches.partial` | Number of partial batches processed +`distsender.batches` | Number of batches processed +`distsender.errors.notleaseholder` | Number of NotLeaseHolderErrors encountered +`distsender.rpc.sent.local` | Number of local RPCs sent +`distsender.rpc.sent.nextreplicaerror` | Number of RPCs sent due to per-replica errors +`distsender.rpc.sent` | Number of RPCs sent +`exec.error` | Number of batch KV requests that failed to execute on this node +`exec.latency` | Latency in nanoseconds of batch KV requests executed on this node +`exec.success` | Number of batch KV requests executed successfully on this node +`gcbytesage` | Cumulative age of non-live data in seconds +`gossip.bytes.received` | Number of received gossip bytes +`gossip.bytes.sent` | Number of sent gossip bytes +`gossip.connections.incoming` | Number of active incoming gossip connections +`gossip.connections.outgoing` | Number of active outgoing gossip connections +`gossip.connections.refused` | Number of refused incoming gossip connections +`gossip.infos.received` | Number of received gossip Info objects +`gossip.infos.sent` | Number of sent gossip Info objects +`intentage` | Cumulative age of intents in seconds +`intentbytes` | Number of bytes in intent KV pairs +`intentcount` | Count of intent keys +`keybytes` | Number of bytes taken up by keys +`keycount` | Count of all keys +`lastupdatenanos` | Time in nanoseconds since Unix epoch at which bytes/keys/intents metrics were last updated +`leases.epoch` | Number of replica leaseholders using epoch-based leases +`leases.error` | Number of failed lease requests +`leases.expiration` | Number of replica leaseholders using expiration-based leases +`leases.success` | Number of successful lease requests +`leases.transfers.error` | Number of failed lease transfers +`leases.transfers.success` | Number of successful lease transfers +`livebytes` | Number of bytes of live data (keys plus values) +`livecount` | Count of live keys +`liveness.epochincrements` | Number of times this node has incremented its liveness epoch +`liveness.heartbeatfailures` | Number of failed node liveness heartbeats from this node +`liveness.heartbeatlatency` | Node liveness heartbeat latency in nanoseconds +`liveness.heartbeatsuccesses` | Number of successful node liveness heartbeats from this node +`liveness.livenodes` | Number of live nodes in the cluster (will be 0 if this node is not itself live) +`node-id` | node ID with labels for advertised RPC and HTTP addresses +`queue.consistency.pending` | Number of pending replicas in the consistency checker queue +`queue.consistency.process.failure` | Number of replicas which failed processing in the consistency checker queue +`queue.consistency.process.success` | Number of replicas successfully processed by the consistency checker queue +`queue.consistency.processingnanos` | Nanoseconds spent processing replicas in the consistency checker queue +`queue.gc.info.abortspanconsidered` | Number of AbortSpan entries old enough to be considered for removal +`queue.gc.info.abortspangcnum` | Number of AbortSpan entries fit for removal +`queue.gc.info.abortspanscanned` | Number of transactions present in the AbortSpan scanned from the engine +`queue.gc.info.intentsconsidered` | Number of 'old' intents +`queue.gc.info.intenttxns` | Number of associated distinct transactions +`queue.gc.info.numkeysaffected` | Number of keys with GC'able data +`queue.gc.info.pushtxn` | Number of attempted pushes +`queue.gc.info.resolvesuccess` | Number of successful intent resolutions +`queue.gc.info.resolvetotal` | Number of attempted intent resolutions +`queue.gc.info.transactionspangcaborted` | Number of GC'able entries corresponding to aborted txns +`queue.gc.info.transactionspangccommitted` | Number of GC'able entries corresponding to committed txns +`queue.gc.info.transactionspangcpending` | Number of GC'able entries corresponding to pending txns +`queue.gc.info.transactionspanscanned` | Number of entries in transaction spans scanned from the engine +`queue.gc.pending` | Number of pending replicas in the GC queue +`queue.gc.process.failure` | Number of replicas which failed processing in the GC queue +`queue.gc.process.success` | Number of replicas successfully processed by the GC queue +`queue.gc.processingnanos` | Nanoseconds spent processing replicas in the GC queue +`queue.raftlog.pending` | Number of pending replicas in the Raft log queue +`queue.raftlog.process.failure` | Number of replicas which failed processing in the Raft log queue +`queue.raftlog.process.success` | Number of replicas successfully processed by the Raft log queue +`queue.raftlog.processingnanos` | Nanoseconds spent processing replicas in the Raft log queue +`queue.raftsnapshot.pending` | Number of pending replicas in the Raft repair queue +`queue.raftsnapshot.process.failure` | Number of replicas which failed processing in the Raft repair queue +`queue.raftsnapshot.process.success` | Number of replicas successfully processed by the Raft repair queue +`queue.raftsnapshot.processingnanos` | Nanoseconds spent processing replicas in the Raft repair queue +`queue.replicagc.pending` | Number of pending replicas in the replica GC queue +`queue.replicagc.process.failure` | Number of replicas which failed processing in the replica GC queue +`queue.replicagc.process.success` | Number of replicas successfully processed by the replica GC queue +`queue.replicagc.processingnanos` | Nanoseconds spent processing replicas in the replica GC queue +`queue.replicagc.removereplica` | Number of replica removals attempted by the replica gc queue +`queue.replicate.addreplica` | Number of replica additions attempted by the replicate queue +`queue.replicate.pending` | Number of pending replicas in the replicate queue +`queue.replicate.process.failure` | Number of replicas which failed processing in the replicate queue +`queue.replicate.process.success` | Number of replicas successfully processed by the replicate queue +`queue.replicate.processingnanos` | Nanoseconds spent processing replicas in the replicate queue +`queue.replicate.purgatory` | Number of replicas in the replicate queue's purgatory, awaiting allocation options +`queue.replicate.rebalancereplica` | Number of replica rebalancer-initiated additions attempted by the replicate queue +`queue.replicate.removedeadreplica` | Number of dead replica removals attempted by the replicate queue (typically in response to a node outage) +`queue.replicate.removereplica` | Number of replica removals attempted by the replicate queue (typically in response to a rebalancer-initiated addition) +`queue.replicate.transferlease` | Number of range lease transfers attempted by the replicate queue +`queue.split.pending` | Number of pending replicas in the split queue +`queue.split.process.failure` | Number of replicas which failed processing in the split queue +`queue.split.process.success` | Number of replicas successfully processed by the split queue +`queue.split.processingnanos` | Nanoseconds spent processing replicas in the split queue +`queue.tsmaintenance.pending` | Number of pending replicas in the time series maintenance queue +`queue.tsmaintenance.process.failure` | Number of replicas which failed processing in the time series maintenance queue +`queue.tsmaintenance.process.success` | Number of replicas successfully processed by the time series maintenance queue +`queue.tsmaintenance.processingnanos` | Nanoseconds spent processing replicas in the time series maintenance queue +`raft.commandsapplied` | Count of Raft commands applied +`raft.enqueued.pending` | Number of pending outgoing messages in the Raft Transport queue +`raft.heartbeats.pending` | Number of pending heartbeats and responses waiting to be coalesced +`raft.process.commandcommit.latency` | Latency histogram in nanoseconds for committing Raft commands +`raft.process.logcommit.latency` | Latency histogram in nanoseconds for committing Raft log entries +`raft.process.tickingnanos` | Nanoseconds spent in store.processRaft() processing replica.Tick() +`raft.process.workingnanos` | Nanoseconds spent in store.processRaft() working +`raft.rcvd.app` | Number of MsgApp messages received by this store +`raft.rcvd.appresp` | Number of MsgAppResp messages received by this store +`raft.rcvd.dropped` | Number of dropped incoming Raft messages +`raft.rcvd.heartbeat` | Number of (coalesced, if enabled) MsgHeartbeat messages received by this store +`raft.rcvd.heartbeatresp` | Number of (coalesced, if enabled) MsgHeartbeatResp messages received by this store +`raft.rcvd.prevote` | Number of MsgPreVote messages received by this store +`raft.rcvd.prevoteresp` | Number of MsgPreVoteResp messages received by this store +`raft.rcvd.prop` | Number of MsgProp messages received by this store +`raft.rcvd.snap` | Number of MsgSnap messages received by this store +`raft.rcvd.timeoutnow` | Number of MsgTimeoutNow messages received by this store +`raft.rcvd.transferleader` | Number of MsgTransferLeader messages received by this store +`raft.rcvd.vote` | Number of MsgVote messages received by this store +`raft.rcvd.voteresp` | Number of MsgVoteResp messages received by this store +`raft.ticks` | Number of Raft ticks queued +`raftlog.behind` | Number of Raft log entries followers on other stores are behind +`raftlog.truncated` | Number of Raft log entries truncated +`range.adds` | Number of range additions +`range.raftleadertransfers` | Number of raft leader transfers +`range.removes` | Number of range removals +`range.snapshots.generated` | Number of generated snapshots +`range.snapshots.normal-applied` | Number of applied snapshots +`range.snapshots.preemptive-applied` | Number of applied pre-emptive snapshots +`range.splits` | Number of range splits +`ranges.unavailable` | Number of ranges with fewer live replicas than needed for quorum +`ranges.underreplicated` | Number of ranges with fewer live replicas than the replication target +`ranges` | Number of ranges +`rebalancing.writespersecond` | Number of keys written (i.e., applied by raft) per second to the store, averaged over a large time period as used in rebalancing decisions +`replicas.commandqueue.combinedqueuesize` | Number of commands in all CommandQueues combined +`replicas.commandqueue.combinedreadcount` | Number of read-only commands in all CommandQueues combined +`replicas.commandqueue.combinedwritecount` | Number of read-write commands in all CommandQueues combined +`replicas.commandqueue.maxoverlaps` | Largest number of overlapping commands seen when adding to any CommandQueue +`replicas.commandqueue.maxreadcount` | Largest number of read-only commands in any CommandQueue +`replicas.commandqueue.maxsize` | Largest number of commands in any CommandQueue +`replicas.commandqueue.maxtreesize` | Largest number of intervals in any CommandQueue's interval tree +`replicas.commandqueue.maxwritecount` | Largest number of read-write commands in any CommandQueue +`replicas.leaders_not_leaseholders` | Number of replicas that are Raft leaders whose range lease is held by another store +`replicas.leaders` | Number of raft leaders +`replicas.leaseholders` | Number of lease holders +`replicas.quiescent` | Number of quiesced replicas +`replicas.reserved` | Number of replicas reserved for snapshots +`replicas` | Number of replicas +`requests.backpressure.split` | Number of backpressured writes waiting on a Range split +`requests.slow.commandqueue` | Number of requests that have been stuck for a long time in the command queue +`requests.slow.distsender` | Number of requests that have been stuck for a long time in the dist sender +`requests.slow.lease` | Number of requests that have been stuck for a long time acquiring a lease +`requests.slow.raft` | Number of requests that have been stuck for a long time in raft +`rocksdb.block.cache.hits` | Count of block cache hits +`rocksdb.block.cache.misses` | Count of block cache misses +`rocksdb.block.cache.pinned-usage` | Bytes pinned by the block cache +`rocksdb.block.cache.usage` | Bytes used by the block cache +`rocksdb.bloom.filter.prefix.checked` | Number of times the bloom filter was checked +`rocksdb.bloom.filter.prefix.useful` | Number of times the bloom filter helped avoid iterator creation +`rocksdb.compactions` | Number of table compactions +`rocksdb.flushes` | Number of table flushes +`rocksdb.memtable.total-size` | Current size of memtable in bytes +`rocksdb.num-sstables` | Number of rocksdb SSTables +`rocksdb.read-amplification` | Number of disk reads per query +`rocksdb.table-readers-mem-estimate` | Memory used by index and filter blocks +`round-trip-latency` | Distribution of round-trip latencies with other nodes in nanoseconds +`security.certificate.expiration.ca` | Expiration timestamp in seconds since Unix epoch for the CA certificate. 0 means no certificate or error. +`security.certificate.expiration.node` | Expiration timestamp in seconds since Unix epoch for the node certificate. 0 means no certificate or error. +`sql.bytesin` | Number of sql bytes received +`sql.bytesout` | Number of sql bytes sent +`sql.conns` | Number of active sql connections +`sql.ddl.count` | Number of SQL DDL statements +`sql.delete.count` | Number of SQL DELETE statements +`sql.distsql.exec.latency` | Latency in nanoseconds of DistSQL statement execution +`sql.distsql.flows.active` | Number of distributed SQL flows currently active +`sql.distsql.flows.total` | Number of distributed SQL flows executed +`sql.distsql.queries.active` | Number of distributed SQL queries currently active +`sql.distsql.queries.total` | Number of distributed SQL queries executed +`sql.distsql.select.count` | Number of DistSQL SELECT statements +`sql.distsql.service.latency` | Latency in nanoseconds of DistSQL request execution +`sql.exec.latency` | Latency in nanoseconds of SQL statement execution +`sql.insert.count` | Number of SQL INSERT statements +`sql.mem.current` | Current sql statement memory usage +`sql.mem.distsql.current` | Current sql statement memory usage for distsql +`sql.mem.distsql.max` | Memory usage per sql statement for distsql +`sql.mem.max` | Memory usage per sql statement +`sql.mem.session.current` | Current sql session memory usage +`sql.mem.session.max` | Memory usage per sql session +`sql.mem.txn.current` | Current sql transaction memory usage +`sql.mem.txn.max` | Memory usage per sql transaction +`sql.misc.count` | Number of other SQL statements +`sql.query.count` | Number of SQL queries +`sql.select.count` | Number of SQL SELECT statements +`sql.service.latency` | Latency in nanoseconds of SQL request execution +`sql.txn.abort.count` | Number of SQL transaction ABORT statements +`sql.txn.begin.count` | Number of SQL transaction BEGIN statements +`sql.txn.commit.count` | Number of SQL transaction COMMIT statements +`sql.txn.rollback.count` | Number of SQL transaction ROLLBACK statements +`sql.update.count` | Number of SQL UPDATE statements +`sys.cgo.allocbytes` | Current bytes of memory allocated by cgo +`sys.cgo.totalbytes` | Total bytes of memory allocated by cgo, but not released +`sys.cgocalls` | Total number of cgo call +`sys.cpu.sys.ns` | Total system cpu time in nanoseconds +`sys.cpu.sys.percent` | Current system cpu percentage +`sys.cpu.user.ns` | Total user cpu time in nanoseconds +`sys.cpu.user.percent` | Current user cpu percentage +`sys.fd.open` | Process open file descriptors +`sys.fd.softlimit` | Process open FD soft limit +`sys.gc.count` | Total number of GC runs +`sys.gc.pause.ns` | Total GC pause in nanoseconds +`sys.gc.pause.percent` | Current GC pause percentage +`sys.go.allocbytes` | Current bytes of memory allocated by go +`sys.go.totalbytes` | Total bytes of memory allocated by go, but not released +`sys.goroutines` | Current number of goroutines +`sys.rss` | Current process RSS +`sys.uptime` | Process uptime in seconds +`sysbytes` | Number of bytes in system KV pairs +`syscount` | Count of system KV pairs +`timeseries.write.bytes` | Total size in bytes of metric samples written to disk +`timeseries.write.errors` | Total errors encountered while attempting to write metrics to disk +`timeseries.write.samples` | Total number of metric samples written to disk +`totalbytes` | Total number of bytes taken up by keys and values including non-live data +`tscache.skl.read.pages` | Number of pages in the read timestamp cache +`tscache.skl.read.rotations` | Number of page rotations in the read timestamp cache +`tscache.skl.write.pages` | Number of pages in the write timestamp cache +`tscache.skl.write.rotations` | Number of page rotations in the write timestamp cache +`txn.abandons` | Number of abandoned KV transactions +`txn.aborts` | Number of aborted KV transactions +`txn.autoretries` | Number of automatic retries to avoid serializable restarts +`txn.commits1PC` | Number of committed one-phase KV transactions +`txn.commits` | Number of committed KV transactions (including 1PC) +`txn.durations` | KV transaction durations in nanoseconds +`txn.restarts.deleterange` | Number of restarts due to a forwarded commit timestamp and a DeleteRange command +`txn.restarts.possiblereplay` | Number of restarts due to possible replays of command batches at the storage layer +`txn.restarts.serializable` | Number of restarts due to a forwarded commit timestamp and isolation=SERIALIZABLE +`txn.restarts.writetooold` | Number of restarts due to a concurrent writer committing first +`txn.restarts` | Number of restarted KV transactions +`valbytes` | Number of bytes taken up by values +`valcount` | Count of all values diff --git a/_includes/v20.2/misc/available-capacity-metric.md b/_includes/v20.2/misc/available-capacity-metric.md new file mode 100644 index 00000000000..61dbcb9cbf2 --- /dev/null +++ b/_includes/v20.2/misc/available-capacity-metric.md @@ -0,0 +1 @@ +If you are testing your deployment locally with multiple CockroachDB nodes running on a single machine (this is [not recommended in production](recommended-production-settings.html#topology)), you must explicitly [set the store size](cockroach-start.html#store) per node in order to display the correct capacity. Otherwise, the machine's actual disk capacity will be counted as a separate store for each node, thus inflating the computed capacity. \ No newline at end of file diff --git a/_includes/v20.2/misc/aws-locations.md b/_includes/v20.2/misc/aws-locations.md new file mode 100644 index 00000000000..8b073c1f230 --- /dev/null +++ b/_includes/v20.2/misc/aws-locations.md @@ -0,0 +1,18 @@ +| Location | SQL Statement | +| ------ | ------ | +| US East (N. Virginia) | `INSERT into system.locations VALUES ('region', 'us-east-1', 37.478397, -76.453077)`| +| US East (Ohio) | `INSERT into system.locations VALUES ('region', 'us-east-2', 40.417287, -76.453077)` | +| US West (N. California) | `INSERT into system.locations VALUES ('region', 'us-west-1', 38.837522, -120.895824)` | +| US West (Oregon) | `INSERT into system.locations VALUES ('region', 'us-west-2', 43.804133, -120.554201)` | +| Canada (Central) | `INSERT into system.locations VALUES ('region', 'ca-central-1', 56.130366, -106.346771)` | +| EU (Frankfurt) | `INSERT into system.locations VALUES ('region', 'eu-central-1', 50.110922, 8.682127)` | +| EU (Ireland) | `INSERT into system.locations VALUES ('region', 'eu-west-1', 53.142367, -7.692054)` | +| EU (London) | `INSERT into system.locations VALUES ('region', 'eu-west-2', 51.507351, -0.127758)` | +| EU (Paris) | `INSERT into system.locations VALUES ('region', 'eu-west-3', 48.856614, 2.352222)` | +| Asia Pacific (Tokyo) | `INSERT into system.locations VALUES ('region', 'ap-northeast-1', 35.689487, 139.691706)` | +| Asia Pacific (Seoul) | `INSERT into system.locations VALUES ('region', 'ap-northeast-2', 37.566535, 126.977969)` | +| Asia Pacific (Osaka-Local) | `INSERT into system.locations VALUES ('region', 'ap-northeast-3', 34.693738, 135.502165)` | +| Asia Pacific (Singapore) | `INSERT into system.locations VALUES ('region', 'ap-southeast-1', 1.352083, 103.819836)` | +| Asia Pacific (Sydney) | `INSERT into system.locations VALUES ('region', 'ap-southeast-2', -33.86882, 151.209296)` | +| Asia Pacific (Mumbai) | `INSERT into system.locations VALUES ('region', 'ap-south-1', 19.075984, 72.877656)` | +| South America (São Paulo) | `INSERT into system.locations VALUES ('region', 'sa-east-1', -23.55052, -46.633309)` | diff --git a/_includes/v20.2/misc/azure-locations.md b/_includes/v20.2/misc/azure-locations.md new file mode 100644 index 00000000000..7119ff8b7cb --- /dev/null +++ b/_includes/v20.2/misc/azure-locations.md @@ -0,0 +1,30 @@ +| Location | SQL Statement | +| -------- | ------------- | +| eastasia (East Asia) | `INSERT into system.locations VALUES ('region', 'eastasia', 22.267, 114.188)` | +| southeastasia (Southeast Asia) | `INSERT into system.locations VALUES ('region', 'southeastasia', 1.283, 103.833)` | +| centralus (Central US) | `INSERT into system.locations VALUES ('region', 'centralus', 41.5908, -93.6208)` | +| eastus (East US) | `INSERT into system.locations VALUES ('region', 'eastus', 37.3719, -79.8164)` | +| eastus2 (East US 2) | `INSERT into system.locations VALUES ('region', 'eastus2', 36.6681, -78.3889)` | +| westus (West US) | `INSERT into system.locations VALUES ('region', 'westus', 37.783, -122.417)` | +| northcentralus (North Central US) | `INSERT into system.locations VALUES ('region', 'northcentralus', 41.8819, -87.6278)` | +| southcentralus (South Central US) | `INSERT into system.locations VALUES ('region', 'southcentralus', 29.4167, -98.5)` | +| northeurope (North Europe) | `INSERT into system.locations VALUES ('region', 'northeurope', 53.3478, -6.2597)` | +| westeurope (West Europe) | `INSERT into system.locations VALUES ('region', 'westeurope', 52.3667, 4.9)` | +| japanwest (Japan West) | `INSERT into system.locations VALUES ('region', 'japanwest', 34.6939, 135.5022)` | +| japaneast (Japan East) | `INSERT into system.locations VALUES ('region', 'japaneast', 35.68, 139.77)` | +| brazilsouth (Brazil South) | `INSERT into system.locations VALUES ('region', 'brazilsouth', -23.55, -46.633)` | +| australiaeast (Australia East) | `INSERT into system.locations VALUES ('region', 'australiaeast', -33.86, 151.2094)` | +| australiasoutheast (Australia Southeast) | `INSERT into system.locations VALUES ('region', 'australiasoutheast', -37.8136, 144.9631)` | +| southindia (South India) | `INSERT into system.locations VALUES ('region', 'southindia', 12.9822, 80.1636)` | +| centralindia (Central India) | `INSERT into system.locations VALUES ('region', 'centralindia', 18.5822, 73.9197)` | +| westindia (West India) | `INSERT into system.locations VALUES ('region', 'westindia', 19.088, 72.868)` | +| canadacentral (Canada Central) | `INSERT into system.locations VALUES ('region', 'canadacentral', 43.653, -79.383)` | +| canadaeast (Canada East) | `INSERT into system.locations VALUES ('region', 'canadaeast', 46.817, -71.217)` | +| uksouth (UK South) | `INSERT into system.locations VALUES ('region', 'uksouth', 50.941, -0.799)` | +| ukwest (UK West) | `INSERT into system.locations VALUES ('region', 'ukwest', 53.427, -3.084)` | +| westcentralus (West Central US) | `INSERT into system.locations VALUES ('region', 'westcentralus', 40.890, -110.234)` | +| westus2 (West US 2) | `INSERT into system.locations VALUES ('region', 'westus2', 47.233, -119.852)` | +| koreacentral (Korea Central) | `INSERT into system.locations VALUES ('region', 'koreacentral', 37.5665, 126.9780)` | +| koreasouth (Korea South) | `INSERT into system.locations VALUES ('region', 'koreasouth', 35.1796, 129.0756)` | +| francecentral (France Central) | `INSERT into system.locations VALUES ('region', 'francecentral', 46.3772, 2.3730)` | +| francesouth (France South) | `INSERT into system.locations VALUES ('region', 'francesouth', 43.8345, 2.1972)` | diff --git a/_includes/v20.2/misc/basic-terms.md b/_includes/v20.2/misc/basic-terms.md new file mode 100644 index 00000000000..be108648c8b --- /dev/null +++ b/_includes/v20.2/misc/basic-terms.md @@ -0,0 +1,9 @@ +Term | Definition +-----|------------ +**Cluster** | Your CockroachDB deployment, which acts as a single logical application. +**Node** | An individual machine running CockroachDB. Many nodes join together to create your cluster. +**Range** | CockroachDB stores all user data (tables, indexes, etc.) and almost all system data in a giant sorted map of key-value pairs. This keyspace is divided into "ranges", contiguous chunks of the keyspace, so that every key can always be found in a single range.

From a SQL perspective, a table and its secondary indexes initially map to a single range, where each key-value pair in the range represents a single row in the table (also called the primary index because the table is sorted by the primary key) or a single row in a secondary index. As soon as that range reaches 512 MiB in size, it splits into two ranges. This process continues for these new ranges as the table and its indexes continue growing. +**Replica** | CockroachDB replicates each range (3 times by default) and stores each replica on a different node. +**Leaseholder** | For each range, one of the replicas holds the "range lease". This replica, referred to as the "leaseholder", is the one that receives and coordinates all read and write requests for the range.

Unlike writes, read requests access the leaseholder and send the results to the client without needing to coordinate with any of the other range replicas. This reduces the network round trips involved and is possible because the leaseholder is guaranteed to be up-to-date due to the fact that all write requests also go to the leaseholder. +**Raft Leader** | For each range, one of the replicas is the "leader" for write requests. Via the [Raft consensus protocol](https://www.cockroachlabs.com/docs/{{ page.version.version }}/architecture/replication-layer.html#raft), this replica ensures that a majority of replicas (the leader and enough followers) agree, based on their Raft logs, before committing the write. The Raft leader is almost always the same replica as the leaseholder. +**Raft Log** | For each range, a time-ordered log of writes to the range that its replicas have agreed on. This log exists on-disk with each replica and is the range's source of truth for consistent replication. diff --git a/_includes/v20.2/misc/beta-warning.md b/_includes/v20.2/misc/beta-warning.md new file mode 100644 index 00000000000..d326ecc3647 --- /dev/null +++ b/_includes/v20.2/misc/beta-warning.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +**This is a beta feature.** It is currently undergoing continued testing. Please [file a Github issue](https://www.cockroachlabs.com/docs/stable/file-an-issue.html) with us if you identify a bug. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/chrome-localhost.md b/_includes/v20.2/misc/chrome-localhost.md new file mode 100644 index 00000000000..24f9bb159a3 --- /dev/null +++ b/_includes/v20.2/misc/chrome-localhost.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +If you are using Google Chrome, and you are getting an error about not being able to reach `localhost` because its certificate has been revoked, go to chrome://flags/#allow-insecure-localhost, enable "Allow invalid certificates for resources loaded from localhost", and then restart the browser. Enabling this Chrome feature degrades security for all sites running on `localhost`, not just CockroachDB's Admin UI, so be sure to enable the feature only temporarily. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/client-side-intervention-example.md b/_includes/v20.2/misc/client-side-intervention-example.md new file mode 100644 index 00000000000..347a6160dcb --- /dev/null +++ b/_includes/v20.2/misc/client-side-intervention-example.md @@ -0,0 +1,27 @@ +The Python-like pseudocode below shows how to implement an application-level retry loop; it does not require your driver or ORM to implement [advanced retry handling logic](advanced-client-side-transaction-retries.html), so it can be used from any programming language or environment. In particular, your retry loop must: + +- Raise an error if the `max_retries` limit is reached +- Retry on `40001` error codes +- [`COMMIT`](commit-transaction.html) at the end of the `try` block +- Implement [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff) logic as shown below for best performance + +~~~ python +while true: + n++ + if n == max_retries: + throw Error("did not succeed within N retries") + try: + # add logic here to run all your statements + conn.exec('COMMIT') + catch error: + if error.code != "40001": + throw error + else: + # This is a retry error, so we roll back the current transaction + # and sleep for a bit before retrying. The sleep time increases + # for each failed transaction. Adapted from + # https://colintemple.com/2017/03/java-exponential-backoff/ + conn.exec('ROLLBACK'); + sleep_ms = int(((2**n) * 100) + rand( 100 - 1 ) + 1) + sleep(sleep_ms) # Assumes your sleep() takes milliseconds +~~~ diff --git a/_includes/v20.2/misc/customizing-the-savepoint-name.md b/_includes/v20.2/misc/customizing-the-savepoint-name.md new file mode 100644 index 00000000000..ed895f906f3 --- /dev/null +++ b/_includes/v20.2/misc/customizing-the-savepoint-name.md @@ -0,0 +1,5 @@ +Set the `force_savepoint_restart` [session variable](set-vars.html#supported-variables) to `true` to enable using a custom name for the [retry savepoint](advanced-client-side-transaction-retries.html#retry-savepoints). + +Once this variable is set, the [`SAVEPOINT`](savepoint.html) statement will accept any name for the retry savepoint, not just `cockroach_restart`. In addition, it causes every savepoint name to be equivalent to `cockroach_restart`, therefore disallowing the use of [nested transactions](transactions.html#nested-transactions). + +This feature exists to support applications that want to use the [advanced client-side transaction retry protocol](advanced-client-side-transaction-retries.html), but cannot customize the name of savepoints to be `cockroach_restart`. For example, this may be necessary because you are using an ORM that requires its own names for savepoints. diff --git a/_includes/v20.2/misc/debug-subcommands.md b/_includes/v20.2/misc/debug-subcommands.md new file mode 100644 index 00000000000..379047a6441 --- /dev/null +++ b/_includes/v20.2/misc/debug-subcommands.md @@ -0,0 +1,3 @@ +While the `cockroach debug` command has a few subcommands, users are expected to use only the [`zip`](cockroach-debug-zip.html), [`encryption-active-key`](cockroach-debug-encryption-active-key.html), [`merge-logs`](cockroach-debug-merge-logs.html), and [`ballast`](cockroach-debug-ballast.html) subcommands. + +The other `debug` subcommands are useful only to CockroachDB's developers and contributors. diff --git a/_includes/v20.2/misc/delete-statistics.md b/_includes/v20.2/misc/delete-statistics.md new file mode 100644 index 00000000000..a568055e583 --- /dev/null +++ b/_includes/v20.2/misc/delete-statistics.md @@ -0,0 +1,17 @@ +To delete statistics for all tables in all databases: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM system.table_statistics WHERE true; +~~~ + +To delete a named set of statistics (e.g, one named "my_stats"), run a query like the following: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM system.table_statistics WHERE name = 'my_stats'; +~~~ + +After deleting statistics, restart the nodes in your cluster to clear the statistics caches. + +For more information about the `DELETE` statement, see [`DELETE`](delete.html). diff --git a/_includes/v20.2/misc/diagnostics-callout.html b/_includes/v20.2/misc/diagnostics-callout.html new file mode 100644 index 00000000000..a969a8cf152 --- /dev/null +++ b/_includes/v20.2/misc/diagnostics-callout.html @@ -0,0 +1 @@ +{{site.data.alerts.callout_info}}By default, each node of a CockroachDB cluster periodically shares anonymous usage details with Cockroach Labs. For an explanation of the details that get shared and how to opt-out of reporting, see Diagnostics Reporting.{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/drivers.md b/_includes/v20.2/misc/drivers.md new file mode 100644 index 00000000000..871f6a830a4 --- /dev/null +++ b/_includes/v20.2/misc/drivers.md @@ -0,0 +1,18 @@ +{{site.data.alerts.callout_info}} +Applications may encounter incompatibilities when using advanced or obscure features of a driver or ORM with **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. +{{site.data.alerts.end}} + +| App Language | Drivers | ORMs | Support level | +|--------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------+------| +| Python | [psycopg2](build-a-python-app-with-cockroachdb.html) | [SQLAlchemy](build-a-python-app-with-cockroachdb-sqlalchemy.html)
[Django](build-a-python-app-with-cockroachdb-django.html)
[PonyORM](build-a-python-app-with-cockroachdb-pony.html)
[peewee](http://docs.peewee-orm.com/en/latest/peewee/playhouse.html#cockroach-database) | Full | +| Java | [JDBC](build-a-java-app-with-cockroachdb.html) | [Hibernate](build-a-java-app-with-cockroachdb-hibernate.html)
[jOOQ](build-a-java-app-with-cockroachdb-jooq.html) | Full | +| Go | [pq](build-a-go-app-with-cockroachdb.html) | [GORM](build-a-go-app-with-cockroachdb-gorm.html) | Full | +| Ruby | [pg](build-a-ruby-app-with-cockroachdb.html) | [ActiveRecord](build-a-ruby-app-with-cockroachdb-activerecord.html) | Beta | +| Node.js | [pg](build-a-nodejs-app-with-cockroachdb.html) | [Sequelize](build-a-nodejs-app-with-cockroachdb-sequelize.html) | Beta | +| C | [libpq](http://www.postgresql.org/docs/9.5/static/libpq.html) | No ORMs tested | Beta | +| C++ | [libpqxx](build-a-c++-app-with-cockroachdb.html) | No ORMs tested | Beta | +| C# (.NET) | [Npgsql](build-a-csharp-app-with-cockroachdb.html) | No ORMs tested | Beta | +| Clojure | [java.jdbc](build-a-clojure-app-with-cockroachdb.html) | No ORMs tested | Beta | +| PHP | [php-pgsql](build-a-php-app-with-cockroachdb.html) | No ORMs tested | Beta | +| Rust | postgres {% comment %} This link is in HTML instead of Markdown because HTML proofer dies bc of https://github.com/rust-lang/crates.io/issues/163 {% endcomment %} | No ORMs tested | Beta | +| TypeScript | No drivers tested | [TypeORM](https://typeorm.io/#/) | Beta | diff --git a/_includes/v20.2/misc/enterprise-features.md b/_includes/v20.2/misc/enterprise-features.md new file mode 100644 index 00000000000..704a3d32e34 --- /dev/null +++ b/_includes/v20.2/misc/enterprise-features.md @@ -0,0 +1,12 @@ +Feature | Description +--------+------------------------- +[Geo-Partitioning](topology-geo-partitioned-replicas.html) | This feature gives you row-level control of how and where your data is stored to dramatically reduce read and write latencies and assist in meeting regulatory requirements in multi-region deployments. +[Follower Reads](follower-reads.html) | This feature reduces read latency in multi-region deployments by using the closest replica at the expense of reading slightly historical data. +[`BACKUP`](backup.html) | This feature creates full or incremental backups of your cluster's schema and data that are consistent as of a given timestamp, stored on a service such as AWS S3, Google Cloud Storage, NFS, or HTTP storage.

Backups can be locality-aware such that each node writes files only to the backup destination that matches the node's [locality](cockroach-start.html#locality). This is useful for reducing cloud storage data transfer costs by keeping data within cloud regions and complying with data domiciling requirements. +[`RESTORE`](restore.html) | This feature restores your cluster's schemas and data from an enterprise `BACKUP`. +[Change Data Capture](change-data-capture.html) (CDC) | This feature provides efficient, distributed, row-level [change feeds into Apache Kafka](create-changefeed.html) for downstream processing such as reporting, caching, or full-text indexing. +[Node Map](enable-node-map.html) | This feature visualizes the geographical configuration of a cluster by plotting node localities on a world map. +[Locality-Aware Index Selection](cost-based-optimizer.html#preferring-the-nearest-index) | Given [multiple identical indexes](topology-duplicate-indexes.html) that have different locality constraints using [replication zones](configure-replication-zones.html), the cost-based optimizer will prefer the index that is closest to the gateway node that is planning the query. In multi-region deployments, this can lead to performance improvements due to improved data locality and reduced network traffic. +[Encryption at Rest](encryption.html#encryption-at-rest-enterprise) | Supplementing CockroachDB's encryption in flight capabilities, this feature provides transparent encryption of a node's data on the local disk. It allows encryption of all files on disk using AES in counter mode, with all key sizes allowed. +[GSSAPI with Kerberos Authentication](gssapi_authentication.html) | CockroachDB supports the Generic Security Services API (GSSAPI) with Kerberos authentication, which lets you use an external enterprise directory system that supports Kerberos, such as Active Directory. +[`EXPORT`](export.html) | This feature uses the CockroachDB distributed execution engine to quickly get large sets of data out of CockroachDB in a CSV format that can be ingested by downstream systems. diff --git a/_includes/v20.2/misc/experimental-warning.md b/_includes/v20.2/misc/experimental-warning.md new file mode 100644 index 00000000000..d38a9755593 --- /dev/null +++ b/_includes/v20.2/misc/experimental-warning.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +**This is an experimental feature**. The interface and output are subject to change. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/explore-benefits-see-also.md b/_includes/v20.2/misc/explore-benefits-see-also.md new file mode 100644 index 00000000000..72cbc961e3b --- /dev/null +++ b/_includes/v20.2/misc/explore-benefits-see-also.md @@ -0,0 +1,8 @@ +- [Replication & Rebalancing](demo-replication-and-rebalancing.html) +- [Fault Tolerance & Recovery](demo-fault-tolerance-and-recovery.html) +- [Low Latency Multi-Region Deployment](demo-low-latency-multi-region-deployment.html) +- [Serializable Transactions](demo-serializable.html) +- [Cross-Cloud Migration](demo-automatic-cloud-migration.html) +- [Follow-the-Workload](demo-follow-the-workload.html) +- [Orchestration](orchestrate-a-local-cluster-with-kubernetes-insecure.html) +- [JSON Support](demo-json-support.html) diff --git a/_includes/v20.2/misc/external-urls.md b/_includes/v20.2/misc/external-urls.md new file mode 100644 index 00000000000..9242847e5b2 --- /dev/null +++ b/_includes/v20.2/misc/external-urls.md @@ -0,0 +1,48 @@ +~~~ +[scheme]://[host]/[path]?[parameters] +~~~ + +Location | Scheme | Host | Parameters | +|-------------------------------------------------------------+-------------+--------------------------------------------------+---------------------------------------------------------------------------- +Amazon | `s3` | Bucket name | `AUTH` [1](#considerations) (optional; can be `implicit` or `specified`), `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` +Azure | `azure` | N/A (see [Example file URLs](#example-file-urls) | `AZURE_ACCOUNT_KEY`, `AZURE_ACCOUNT_NAME` +Google Cloud [2](#considerations) | `gs` | Bucket name | `AUTH` (optional; can be `default`, `implicit`, or `specified`), `CREDENTIALS` +HTTP [3](#considerations) | `http` | Remote host | N/A +NFS/Local [4](#considerations) | `nodelocal` | `nodeID` or `self` [5](#considerations) (see [Example file URLs](#example-file-urls)) | N/A +S3-compatible services [6](#considerations) | `s3` | Bucket name | `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `AWS_REGION` [7](#considerations) (optional), `AWS_ENDPOINT` + +{{site.data.alerts.callout_info}} +The location parameters often contain special characters that need to be URI-encoded. Use Javascript's [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) function or Go language's [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape) function to URI-encode the parameters. Other languages provide similar functions to URI-encode special characters. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +If your environment requires an HTTP or HTTPS proxy server for outgoing connections, you can set the standard `HTTP_PROXY` and `HTTPS_PROXY` environment variables when starting CockroachDB. + + If you cannot run a full proxy, you can disable external HTTP(S) access (as well as custom HTTP(S) endpoints) when performing bulk operations (e.g., `BACKUP`, `RESTORE`, etc.) by using the [`--external-io-disable-http` flag](cockroach-start.html#security). You can also disable the use of implicit credentials when accessing external cloud storage services for various bulk operations by using the [`--external-io-disable-implicit-credentials` flag](cockroach-start.html#security). +{{site.data.alerts.end}} + + + +- 1 If the `AUTH` parameter is not provided, AWS connections default to `specified` and the access keys must be provided in the URI parameters. If the `AUTH` parameter is `implicit`, the access keys can be ommitted and [the credentials will be loaded from the environment](https://docs.aws.amazon.com/sdk-for-go/api/aws/session/). + +- 2 If the `AUTH` parameter is not specified, the `cloudstorage.gs.default.key` [cluster setting](cluster-settings.html) will be used if it is non-empty, otherwise the `implicit` behavior is used. If the `AUTH` parameter is `implicit`, all GCS connections use Google's [default authentication strategy](https://cloud.google.com/docs/authentication/production#providing_credentials_to_your_application). If the `AUTH` parameter is `default`, the `cloudstorage.gs.default.key` [cluster setting](cluster-settings.html) must be set to the contents of a [service account file](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) which will be used during authentication. If the `AUTH` parameter is `specified`, GCS connections are authenticated on a per-statement basis, which allows the JSON key object to be sent in the `CREDENTIALS` parameter. The JSON key object should be base64-encoded (using the standard encoding in [RFC 4648](https://tools.ietf.org/html/rfc4648)). + +- 3 You can create your own HTTP server with [Caddy or nginx](create-a-file-server.html). A custom root CA can be appended to the system's default CAs by setting the `cloudstorage.http.custom_ca` [cluster setting](cluster-settings.html), which will be used when verifying certificates from HTTPS URLs. + +- 4 The file system backup location on the NFS drive is relative to the path specified by the `--external-io-dir` flag set while [starting the node](cockroach-start.html). If the flag is set to `disabled`, then imports from local directories and NFS drives are disabled. + +- 5 Using a `nodeID` is required and the data files will be in the `extern` directory of the specified node. In most cases (including single-node clusters), using `nodelocal://1/` is sufficient. Use `self` if you do not want to specify a `nodeID`, and the individual data files will be in the `extern` directories of arbitrary nodes; however, to work correctly, each node must have the [`--external-io-dir` flag](cockroach-start.html#general) point to the same NFS mount or other network-backed, shared storage. + +- 6 A custom root CA can be appended to the system's default CAs by setting the `cloudstorage.http.custom_ca` [cluster setting](cluster-settings.html), which will be used when verifying certificates from an S3-compatible service. + +- 7 The `AWS_REGION` parameter is optional since it is not a required parameter for most S3-compatible services. Specify the parameter only if your S3-compatible service requires it. + +#### Example file URLs + +Location | Example +-------------+---------------------------------------------------------------------------------- +Amazon S3 | `s3://acme-co/employees.sql?AWS_ACCESS_KEY_ID=123&AWS_SECRET_ACCESS_KEY=456` +Azure | `azure://employees.sql?AZURE_ACCOUNT_KEY=123&AZURE_ACCOUNT_NAME=acme-co` +Google Cloud | `gs://acme-co/employees.sql` +HTTP | `http://localhost:8080/employees.sql` +NFS/Local | `nodelocal://1/path/employees`, `nodelocal://self/nfsmount/backups/employees` [5](#considerations) diff --git a/_includes/v20.2/misc/force-index-selection.md b/_includes/v20.2/misc/force-index-selection.md new file mode 100644 index 00000000000..cc9798bdd7d --- /dev/null +++ b/_includes/v20.2/misc/force-index-selection.md @@ -0,0 +1,61 @@ +By using the explicit index annotation, you can override [CockroachDB's index selection](https://www.cockroachlabs.com/blog/index-selection-cockroachdb-2/) and use a specific [index](indexes.html) when reading from a named table. + +{{site.data.alerts.callout_info}} +Index selection can impact [performance](performance-best-practices-overview.html), but does not change the result of a query. +{{site.data.alerts.end}} + +The syntax to force a scan of a specific index is: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM table@my_idx; +~~~ + +This is equivalent to the longer expression: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM table@{FORCE_INDEX=my_idx}; +~~~ + +The syntax to force a **reverse scan** of a specific index is: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM table@{FORCE_INDEX=my_idx,DESC}; +~~~ + +Forcing a reverse can is sometimes useful during [performance tuning](performance-best-practices-overview.html). For reference, the full syntax for choosing an index and its scan direction is + +{% include copy-clipboard.html %} +~~~ sql +SELECT * FROM table@{FORCE_INDEX=idx[,DIRECTION]} +~~~ + +where the optional `DIRECTION` is either `ASC` (ascending) or `DESC` (descending). + +When a direction is specified, that scan direction is forced; otherwise the [cost-based optimizer](cost-based-optimizer.html) is free to choose the direction it calculates will result in the best performance. + +You can verify that the optimizer is choosing your desired scan direction using [`EXPLAIN (OPT)`](explain.html#opt-option). For example, given the table + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE kv (K INT PRIMARY KEY, v INT); +~~~ + +you can check the scan direction with: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (opt) SELECT * FROM users@{FORCE_INDEX=primary,DESC}; +~~~ + +~~~ + text ++-------------------------------------+ + scan users,rev + └── flags: force-index=primary,rev +(2 rows) +~~~ + +To see all indexes available on a table, use [`SHOW INDEXES`](show-index.html). diff --git a/_includes/v20.2/misc/gce-locations.md b/_includes/v20.2/misc/gce-locations.md new file mode 100644 index 00000000000..22122aae78d --- /dev/null +++ b/_includes/v20.2/misc/gce-locations.md @@ -0,0 +1,18 @@ +| Location | SQL Statement | +| ------ | ------ | +| us-east1 (South Carolina) | `INSERT into system.locations VALUES ('region', 'us-east1', 33.836082, -81.163727)` | +| us-east4 (N. Virginia) | `INSERT into system.locations VALUES ('region', 'us-east4', 37.478397, -76.453077)` | +| us-central1 (Iowa) | `INSERT into system.locations VALUES ('region', 'us-central1', 42.032974, -93.581543)` | +| us-west1 (Oregon) | `INSERT into system.locations VALUES ('region', 'us-west1', 43.804133, -120.554201)` | +| northamerica-northeast1 (Montreal) | `INSERT into system.locations VALUES ('region', 'northamerica-northeast1', 56.130366, -106.346771)` | +| europe-west1 (Belgium) | `INSERT into system.locations VALUES ('region', 'europe-west1', 50.44816, 3.81886)` | +| europe-west2 (London) | `INSERT into system.locations VALUES ('region', 'europe-west2', 51.507351, -0.127758)` | +| europe-west3 (Frankfurt) | `INSERT into system.locations VALUES ('region', 'europe-west3', 50.110922, 8.682127)` | +| europe-west4 (Netherlands) | `INSERT into system.locations VALUES ('region', 'europe-west4', 53.4386, 6.8355)` | +| europe-west6 (Zürich) | `INSERT into system.locations VALUES ('region', 'europe-west6', 47.3769, 8.5417)` | +| asia-east1 (Taiwan) | `INSERT into system.locations VALUES ('region', 'asia-east1', 24.0717, 120.5624)` | +| asia-northeast1 (Tokyo) | `INSERT into system.locations VALUES ('region', 'asia-northeast1', 35.689487, 139.691706)` | +| asia-southeast1 (Singapore) | `INSERT into system.locations VALUES ('region', 'asia-southeast1', 1.352083, 103.819836)` | +| australia-southeast1 (Sydney) | `INSERT into system.locations VALUES ('region', 'australia-southeast1', -33.86882, 151.209296)` | +| asia-south1 (Mumbai) | `INSERT into system.locations VALUES ('region', 'asia-south1', 19.075984, 72.877656)` | +| southamerica-east1 (São Paulo) | `INSERT into system.locations VALUES ('region', 'southamerica-east1', -23.55052, -46.633309)` | diff --git a/_includes/v20.2/misc/haproxy.md b/_includes/v20.2/misc/haproxy.md new file mode 100644 index 00000000000..375af8e937d --- /dev/null +++ b/_includes/v20.2/misc/haproxy.md @@ -0,0 +1,39 @@ +By default, the generated configuration file is called `haproxy.cfg` and looks as follows, with the `server` addresses pre-populated correctly: + + ~~~ + global + maxconn 4096 + + defaults + mode tcp + # Timeout values should be configured for your specific use. + # See: https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20connect + timeout connect 10s + timeout client 1m + timeout server 1m + # TCP keep-alive on client side. Server already enables them. + option clitcpka + + listen psql + bind :26257 + mode tcp + balance roundrobin + option httpchk GET /health?ready=1 + server cockroach1 :26257 check port 8080 + server cockroach2 :26257 check port 8080 + server cockroach3 :26257 check port 8080 + ~~~ + + The file is preset with the minimal [configurations](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html) needed to work with your running cluster: + + Field | Description + ------|------------ + `timeout connect`
`timeout client`
`timeout server` | Timeout values that should be suitable for most deployments. + `bind` | The port that HAProxy listens on. This is the port clients will connect to and thus needs to be allowed by your network configuration.

This tutorial assumes HAProxy is running on a separate machine from CockroachDB nodes. If you run HAProxy on the same machine as a node (not recommended), you'll need to change this port, as `26257` is likely already being used by the CockroachDB node. + `balance` | The balancing algorithm. This is set to `roundrobin` to ensure that connections get rotated amongst nodes (connection 1 on node 1, connection 2 on node 2, etc.). Check the [HAProxy Configuration Manual](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html#4-balance) for details about this and other balancing algorithms. + `option httpchk` | The HTTP endpoint that HAProxy uses to check node health. [`/health?ready=1`](monitoring-and-alerting.html#health-ready-1) ensures that HAProxy doesn't direct traffic to nodes that are live but not ready to receive requests. + `server` | For each included node, this field specifies the address the node advertises to other nodes in the cluster, i.e., the addressed pass in the [`--advertise-addr` flag](cockroach-start.html#networking) on node startup. Make sure hostnames are resolvable and IP addresses are routable from HAProxy. + + {{site.data.alerts.callout_info}} + For full details on these and other configuration settings, see the [HAProxy Configuration Manual](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html). + {{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/install-next-steps.html b/_includes/v20.2/misc/install-next-steps.html new file mode 100644 index 00000000000..2111bdbed9c --- /dev/null +++ b/_includes/v20.2/misc/install-next-steps.html @@ -0,0 +1,16 @@ + diff --git a/_includes/v20.2/misc/linux-binary-prereqs.md b/_includes/v20.2/misc/linux-binary-prereqs.md new file mode 100644 index 00000000000..541183fe71b --- /dev/null +++ b/_includes/v20.2/misc/linux-binary-prereqs.md @@ -0,0 +1 @@ +

The CockroachDB binary for Linux requires glibc, libncurses, and tzdata, which are found by default on nearly all Linux distributions, with Alpine as the notable exception.

diff --git a/_includes/v20.2/misc/logging-flags.md b/_includes/v20.2/misc/logging-flags.md new file mode 100644 index 00000000000..02a800a54bb --- /dev/null +++ b/_includes/v20.2/misc/logging-flags.md @@ -0,0 +1,9 @@ +Flag | Description +-----|------------ +`--log-dir` | Enable logging to files and write logs to the specified directory.

Setting `--log-dir` to a blank directory (`--log-dir=""`) disables logging to files. +`--log-dir-max-size` | After the log directory reaches the specified size, delete the oldest log file. The flag's argument takes standard file sizes, such as `--log-dir-max-size=1GiB`.

**Default**: 100MiB +`--log-file-max-size` | After logs reach the specified size, begin writing logs to a new file. The flag's argument takes standard file sizes, such as `--log-file-max-size=2MiB`.

**Default**: 10MiB +`--log-file-verbosity` | Only writes messages to log files if they are at or above the specified [severity level](debug-and-error-logs.html#severity-levels), such as `--log-file-verbosity=WARNING`. **Requires** logging to files.

**Default**: `INFO` +`--logtostderr` | Enable logging to `stderr` for messages at or above the specified [severity level](debug-and-error-logs.html#severity-levels), such as `--logtostderr=ERROR`

If you use this flag without specifying the severity level (e.g., `cockroach start --logtostderr`), it prints messages of *all* severities to `stderr`.

Setting `--logtostderr=NONE` disables logging to `stderr`. +`--no-color` | Do not colorize `stderr`. Possible values: `true` or `false`.

When set to `false`, messages logged to `stderr` are colorized based on [severity level](debug-and-error-logs.html#severity-levels).

**Default:** `false` +`--sql-audit-dir` | New in v2.0: If non-empty, create a SQL audit log in this directory. By default, SQL audit logs are written in the same directory as the other logs generated by CockroachDB. For more information, see [SQL Audit Logging](sql-audit-logging.html). diff --git a/_includes/v20.2/misc/mitigate-contention-note.md b/_includes/v20.2/misc/mitigate-contention-note.md new file mode 100644 index 00000000000..ffe3cff554a --- /dev/null +++ b/_includes/v20.2/misc/mitigate-contention-note.md @@ -0,0 +1,5 @@ +{{site.data.alerts.callout_info}} +It's possible to mitigate read-write contention and reduce transaction retries using the following techniques: +1. By performing reads using [`AS OF SYSTEM TIME`](performance-best-practices-overview.html#use-as-of-system-time-to-decrease-conflicts-with-long-running-queries). +2. By using [`SELECT FOR UPDATE`](select-for-update.html) to order transactions by controlling concurrent access to one or more rows of a table. This reduces retries in scenarios where a transaction performs a read and then updates the same row it just read. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/movr-schema.md b/_includes/v20.2/misc/movr-schema.md new file mode 100644 index 00000000000..0ce2a8f83e7 --- /dev/null +++ b/_includes/v20.2/misc/movr-schema.md @@ -0,0 +1,12 @@ +The six tables in the `movr` database store user, vehicle, and ride data for MovR: + +Table | Description +--------|---------------------------- +`users` | People registered for the service. +`vehicles` | The pool of vehicles available for the service. +`rides` | When and where users have rented a vehicle. +`promo_codes` | Promotional codes for users. +`user_promo_codes` | Promotional codes in use by users. +`vehicle_location_histories` | Vehicle location history. + +Geo-partitioning schema diff --git a/_includes/v20.2/misc/movr-workflow.md b/_includes/v20.2/misc/movr-workflow.md new file mode 100644 index 00000000000..3dc9a61b910 --- /dev/null +++ b/_includes/v20.2/misc/movr-workflow.md @@ -0,0 +1,49 @@ +The workflow for MovR is as follows (with approximations of the corresponding SQL for each step): + +1. A user loads the app and sees the 25 closest vehicles: + + ~~~ sql + > SELECT id, city, status, ... FROM vehicles WHERE city = + ~~~ + +2. The user signs up for the service: + + ~~~ sql + > INSERT INTO users (id, name, address, ...) VALUES ... + ~~~ + +3. In some cases, the user adds their own vehicle to share: + + ~~~ sql + > INSERT INTO vehicles (id, city, type, ...) VALUES ... + ~~~ + +4. More often, the user reserves a vehicle and starts a ride, applying a promo code, if available and valid: + + ~~~ sql + > SELECT code FROM user_promo_codes WHERE user_id = ... + ~~~ + + ~~~ sql + > UPDATE vehicles SET status = 'in_use' WHERE ... + ~~~ + + ~~~ sql + > INSERT INTO rides (id, city, start_addr, ...) VALUES ... + ~~~ + +5. During the ride, MovR tracks the location of the vehicle: + + ~~~ sql + > INSERT INTO vehicle_location_histories (city, ride_id, timestamp, lat, long) VALUES ... + ~~~ + +6. The user ends the ride and releases the vehicle: + + ~~~ sql + > UPDATE vehicles SET status = 'available' WHERE ... + ~~~ + + ~~~ sql + > UPDATE rides SET end_address = ... + ~~~ diff --git a/_includes/v20.2/misc/multi-store-nodes.md b/_includes/v20.2/misc/multi-store-nodes.md new file mode 100644 index 00000000000..01642597169 --- /dev/null +++ b/_includes/v20.2/misc/multi-store-nodes.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +In the absence of [special replication constraints](configure-replication-zones.html), CockroachDB rebalances replicas to take advantage of available storage capacity. However, in a 3-node cluster with multiple stores per node, CockroachDB is **not** able to rebalance replicas from one store to another store on the same node because this would temporarily result in the node having multiple replicas of the same range, which is not allowed. This is due to the mechanics of rebalancing, where the cluster first creates a copy of the replica at the target destination before removing the source replica. To allow this type of cross-store rebalancing, the cluster must have 4 or more nodes; this allows the cluster to create a copy of the replica on a node that doesn't already have a replica of the range before removing the source replica and then migrating the new replica to the store with more capacity on the original node. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/remove-user-callout.html b/_includes/v20.2/misc/remove-user-callout.html new file mode 100644 index 00000000000..925f83d779d --- /dev/null +++ b/_includes/v20.2/misc/remove-user-callout.html @@ -0,0 +1 @@ +Removing a user does not remove that user's privileges. Therefore, to prevent a future user with an identical username from inheriting an old user's privileges, it's important to revoke a user's privileges before or after removing the user. diff --git a/_includes/v20.2/misc/schema-change-stmt-note.md b/_includes/v20.2/misc/schema-change-stmt-note.md new file mode 100644 index 00000000000..b522b658652 --- /dev/null +++ b/_includes/v20.2/misc/schema-change-stmt-note.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +This statement performs a schema change. For more information about how online schema changes work in CockroachDB, see [Online Schema Changes](online-schema-changes.html). +{{site.data.alerts.end}} diff --git a/_includes/v20.2/misc/schema-change-view-job.md b/_includes/v20.2/misc/schema-change-view-job.md new file mode 100644 index 00000000000..8861174d621 --- /dev/null +++ b/_includes/v20.2/misc/schema-change-view-job.md @@ -0,0 +1 @@ +This schema change statement is registered as a job. You can view long-running jobs with [`SHOW JOBS`](show-jobs.html). diff --git a/_includes/v20.2/misc/session-vars.html b/_includes/v20.2/misc/session-vars.html new file mode 100644 index 00000000000..b6d057fe2b0 --- /dev/null +++ b/_includes/v20.2/misc/session-vars.html @@ -0,0 +1,503 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Variable nameDescriptionInitial valueModify with + SET + ?View with + SHOW + ?
+ application_name + The current application name for statistics collection.Empty string, or cockroach for sessions from the built-in SQL client.YesYes
+ bytea_output + The mode for conversions from STRING to BYTES.hexYesYes
+ crdb_version + The version of CockroachDB.`CockroachDB OSS version`NoYes
+ database + The current database.Database in connection string, or empty if not specified.YesYes
+ default_int_size + The size, in bytes, of an INT type. + 8 + YesYes
+ default_transaction_isolation + All transactions execute with SERIALIZABLE isolation. See Transactions: Isolation levels. + SERIALIZABLE + NoYes
+ default_transaction_read_only + The default transaction access mode for the current session. If set to on, only read operations are allowed in transactions in the current session; if set to off, both read and write operations are allowed. See SET TRANSACTION + for more details. + off + YesYes
+ distsql + The query distribution mode for the session. By default, CockroachDB determines which queries are faster to execute if distributed across multiple nodes, and all other queries are run through the gateway node. + auto + YesYes
+ + enable_implicit_select_for_update + Indicates whether UPDATE statements acquire locks using the FOR UPDATE locking mode during their initial row scan, which improves performance for contended workloads. For more information about how FOR UPDATE locking works, see the documentation for SELECT FOR UPDATE. + on + YesYes
+ enable_zigzag_join + Indicates whether the cost-based optimizer will plan certain queries using a zig-zag merge join algorithm, which searches for the desired intersection by jumping back and forth between the indexes based on the fact that after constraining indexes, they share an ordering. + on + YesYes
+ extra_float_digits + The number of digits displayed for floating-point values. Only values between -15 and 3 are supported. + 0 + YesYes
+ reorder_joins_limit + Maximum number of joins that the optimizer will attempt to reorder when searching for an optimal query execution plan. For more information, see Join reordering. + 4 + YesYes
force_savepoint_restartWhen set to true, allows the SAVEPOINT statement to accept any name for a savepoint. + off + YesYes
+ node_id + The ID of the node currently connected to.
+
This variable is particularly useful for verifying load balanced connections.
Node-dependentNoYes
+ optimizer_foreign_keys + If off, disables optimizer-driven foreign key checks. + onYesYes
+ results_buffer_size + The default size of the buffer that accumulates results for a statement or a batch of statements before they are sent to the client. This can also be set for all connections using the 'sql.defaults.results_buffer_size' cluster setting. Note that auto-retries generally only happen while no results have been delivered to the client, so reducing this size can increase the number of retriable errors a client receives. On the other hand, increasing the buffer size can increase the delay until the client receives the first result row. Setting to 0 disables any buffering. + + 16384 + YesYes
+ require_explicit_primary_keys + If on, CockroachDB throws on error for all tables created without an explicit primary key defined. + + off + YesYes
+ search_path + A list of schemas that will be searched to resolve unqualified table or function names. For more details, see SQL name resolution. + public + YesYes
+ server_version + The version of PostgreSQL that CockroachDB emulates.Version-dependentNoYes
+ server_version_num + The version of PostgreSQL that CockroachDB emulates.Version-dependentYesYes
+ session_id + The ID of the current session.Session-dependentNoYes
+ session_user + The user connected for the current session.User in connection stringNoYes
+ sql_safe_updates + If false, potentially unsafe SQL statements are allowed, including DROP of a non-empty database and all dependent objects, DELETE without a WHERE clause, UPDATE without a WHERE clause, and ALTER TABLE .. DROP COLUMN. See Allow Potentially Unsafe SQL Statements for more details. + true for interactive sessions from the built-in SQL client,
false for sessions from other clients
YesYes
+ statement_timeout + The amount of time a statement can run before being stopped.
+
This value can be an int (e.g., 10) and will be interpreted as milliseconds. It can also be an interval or string argument, where the string can be parsed as a valid interval (e.g., '4s'). A value of 0 turns it off.
+ 0s + YesYes
+ timezone + The default time zone for the current session.
+
This session variable was named "time zone" (with a space) in CockroachDB 1.x. It has been renamed for compatibility with PostgreSQL.
+ UTC + YesYes
+ tracing + The trace recording state. + off + + Yes
+ transaction_isolation + All transactions execute with SERIALIZABLE isolation. See Transactions: Isolation levels.
+
This session variable was called transaction isolation level (with spaces) in CockroachDB 1.x. It has been renamed for compatibility with PostgreSQL.
+ SERIALIZABLE + NoYes
+ transaction_priority + The priority of the current transaction. See Transactions: Isolation levels for more details.
+
This session variable was called transaction priority (with a space) in CockroachDB 1.x. It has been renamed for compatibility with PostgreSQL.
+ NORMAL + YesYes
+ transaction_read_only + The access mode of the current transaction. See Set Transaction for more details. + off + YesYes
+ transaction_status + The state of the current transaction. See Transactions for more details.
+
This session variable was called transaction status (with a space) in CockroachDB 1.x. It has been renamed for compatibility with PostgreSQL.
+ NoTxn + NoYes
+ vectorize + The vectorized execution engine mode. Options include auto, on, and off. + For more details, see Configuring vectorized execution for CockroachDB. + + auto + YesYes
+ vectorize_row_count_threshold + The minimum number of rows required to use the vectorized engine to execute a query plan. + + 1000 + YesYes
+ client_encoding + (Reserved; exposed only for ORM compatibility.) + UTF8 + NoYes
+ client_min_messages + (Reserved; exposed only for ORM compatibility.) + notice + NoYes
+ datestyle + (Reserved; exposed only for ORM compatibility.) + ISO + NoYes
+ integer_datetimes + (Reserved; exposed only for ORM compatibility.) + on + NoYes
+ intervalstyle + (Reserved; exposed only for ORM compatibility.) + postgres + NoYes
+ max_identifier_length + (Reserved; exposed only for ORM compatibility.) + 128 + NoYes
+ max_index_keys + (Reserved; exposed only for ORM compatibility.) + 32 + NoYes
+ standard_conforming_strings + (Reserved; exposed only for ORM compatibility.) + on + NoYes
+ server_encoding + (Reserved; exposed only for ORM compatibility.) + UTF8 + YesYes
diff --git a/_includes/v20.2/misc/sorting-delete-output.md b/_includes/v20.2/misc/sorting-delete-output.md new file mode 100644 index 00000000000..fa0d6e54be7 --- /dev/null +++ b/_includes/v20.2/misc/sorting-delete-output.md @@ -0,0 +1,9 @@ +To sort the output of a `DELETE` statement, use: + +{% include copy-clipboard.html %} +~~~ sql +> WITH a AS (DELETE ... RETURNING ...) + SELECT ... FROM a ORDER BY ... +~~~ + +For an example, see [Sort and return deleted rows](delete.html#sort-and-return-deleted-rows). diff --git a/_includes/v20.2/orchestration/kubernetes-expand-disk-size.md b/_includes/v20.2/orchestration/kubernetes-expand-disk-size.md new file mode 100644 index 00000000000..5f5f77b4962 --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-expand-disk-size.md @@ -0,0 +1,184 @@ +You can expand certain [types of persistent volumes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#types-of-persistent-volumes +) (including GCE Persistent Disk and Amazon Elastic Block Store) by editing their persistent volume claims. Increasing disk size is often beneficial for CockroachDB performance. Read our [Kubernetes performance guide](kubernetes-performance.html#disk-size) for guidance on disks. + +1. Get the persistent volume claims for the volumes: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pvc + ~~~ + +
+ ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + datadir-my-release-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + datadir-my-release-cockroachdb-1 Bound pvc-75e143ca-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + datadir-my-release-cockroachdb-2 Bound pvc-75ef409a-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + ~~~ +
+ +
+ ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + datadir-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + datadir-cockroachdb-1 Bound pvc-75e143ca-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + datadir-cockroachdb-2 Bound pvc-75ef409a-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 17m + ~~~ +
+ +2. In order to expand a persistent volume claim, `AllowVolumeExpansion` in its storage class must be `true`. Examine the storage class: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe storageclass standard + ~~~ + + ~~~ + Name: standard + IsDefaultClass: Yes + Annotations: storageclass.kubernetes.io/is-default-class=true + Provisioner: kubernetes.io/gce-pd + Parameters: type=pd-standard + AllowVolumeExpansion: False + MountOptions: + ReclaimPolicy: Delete + VolumeBindingMode: Immediate + Events: + ~~~ + + If necessary, edit the storage class: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl patch storageclass standard -p '{"allowVolumeExpansion": true}' + ~~~ + + ~~~ + storageclass.storage.k8s.io/standard patched + ~~~ + +3. Edit one of the persistent volume claims to request more space: + + {{site.data.alerts.callout_info}} + The requested `storage` value must be larger than the previous value. You cannot use this method to decrease the disk size. + {{site.data.alerts.end}} + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl patch pvc datadir-my-release-cockroachdb-0 -p '{"spec": {"resources": {"requests": {"storage": "200Gi"}}}}' + ~~~ + + ~~~ + persistentvolumeclaim/datadir-my-release-cockroachdb-0 patched + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl patch pvc datadir-cockroachdb-0 -p '{"spec": {"resources": {"requests": {"storage": "200Gi"}}}}' + ~~~ + + ~~~ + persistentvolumeclaim/datadir-cockroachdb-0 patched + ~~~ +
+ +4. Check the capacity of the persistent volume claim: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pvc datadir-my-release-cockroachdb-0 + ~~~ + + ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + datadir-my-release-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 18m + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pvc datadir-cockroachdb-0 + ~~~ + + ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE + datadir-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 100Gi RWO standard 18m + ~~~ +
+ + If the PVC capacity has not changed, this may be because `AllowVolumeExpansion` was initially set to `false` or because the [volume has a file system](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#resizing-an-in-use-persistentvolumeclaim) that has to be expanded. You will need to start or restart a pod in order to have it reflect the new capacity. + + {{site.data.alerts.callout_success}} + Running `kubectl get pv` will display the persistent volumes with their *requested* capacity and not their actual capacity. This can be misleading, so it's best to use `kubectl get pvc`. + {{site.data.alerts.end}} + +5. Examine the persistent volume claim. If the volume has a file system, you will see a `FileSystemResizePending` condition with an accompanying message: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe pvc datadir-my-release-cockroachdb-0 + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe pvc datadir-cockroachdb-0 + ~~~ +
+ + ~~~ + Waiting for user to (re-)start a pod to finish file system resize of volume on node. + ~~~ + +6. Delete the corresponding pod to restart it: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete pod my-release-cockroachdb-0 + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete pod cockroachdb-0 + ~~~ +
+ + The `FileSystemResizePending` condition and message will be removed. + +7. View the updated persistent volume claim: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pvc datadir-my-release-cockroachdb-0 + ~~~ + + ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +datadir-my-release-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 200Gi RWO standard 20m + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pvc datadir-cockroachdb-0 + ~~~ + + ~~~ + NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE +datadir-cockroachdb-0 Bound pvc-75dadd4c-01a1-11ea-b065-42010a8e00cb 200Gi RWO standard 20m + ~~~ +
+ +8. The CockroachDB cluster needs to be expanded one node at a time. Repeat steps 3 - 6 to increase the capacities of the remaining volumes by the same amount. \ No newline at end of file diff --git a/_includes/v20.2/orchestration/kubernetes-limitations.md b/_includes/v20.2/orchestration/kubernetes-limitations.md new file mode 100644 index 00000000000..264c1e33acc --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-limitations.md @@ -0,0 +1,11 @@ +#### Kubernetes version + +Kubernetes 1.8 or higher is required in order to use our most up-to-date configuration files. Earlier Kubernetes releases do not support some of the options used in our configuration files. If you need to run on an older version of Kubernetes, we have kept around configuration files that work on older Kubernetes releases in the versioned subdirectories of [https://github.com/cockroachdb/cockroach/tree/master/cloud/kubernetes](https://github.com/cockroachdb/cockroach/tree/master/cloud/kubernetes) (e.g., [v1.7](https://github.com/cockroachdb/cockroach/tree/master/cloud/kubernetes/v1.7)). + +#### Helm version + +Helm 3.0 or higher is required when using our instructions to [deploy via Helm](orchestrate-cockroachdb-with-kubernetes.html#step-2-start-cockroachdb). + +#### Storage + +At this time, orchestrations of CockroachDB with Kubernetes use external persistent volumes that are often replicated by the provider. Because CockroachDB already replicates data automatically, this additional layer of replication is unnecessary and can negatively impact performance. High-performance use cases on a private Kubernetes cluster may want to consider a [DaemonSet](kubernetes-performance.html#running-in-a-daemonset) deployment until StatefulSets support node-local storage. \ No newline at end of file diff --git a/_includes/v20.2/orchestration/kubernetes-prometheus-alertmanager.md b/_includes/v20.2/orchestration/kubernetes-prometheus-alertmanager.md new file mode 100644 index 00000000000..98d73cf6059 --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-prometheus-alertmanager.md @@ -0,0 +1,218 @@ +Despite CockroachDB's various [built-in safeguards against failure](high-availability.html), it is critical to actively monitor the overall health and performance of a cluster running in production and to create alerting rules that promptly send notifications when there are events that require investigation or intervention. + +### Configure Prometheus + +Every node of a CockroachDB cluster exports granular timeseries metrics formatted for easy integration with [Prometheus](https://prometheus.io/), an open source tool for storing, aggregating, and querying timeseries data. This section shows you how to orchestrate Prometheus as part of your Kubernetes cluster and pull these metrics into Prometheus for external monitoring. + +This guidance is based on [CoreOS's Prometheus Operator](https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md), which allows a Prometheus instance to be managed using native Kubernetes concepts. + +{{site.data.alerts.callout_info}} +If you're on Hosted GKE, before starting, make sure the email address associated with your Google Cloud account is part of the `cluster-admin` RBAC group, as shown in [Step 1. Start Kubernetes](#hosted-gke). +{{site.data.alerts.end}} + +1. From your local workstation, edit the `cockroachdb` service to add the `prometheus: cockroachdb` label: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl label svc cockroachdb prometheus=cockroachdb + ~~~ + + ~~~ + service/cockroachdb labeled + ~~~ + + This ensures that there is a Prometheus job and monitoring data only for the `cockroachdb` service, not for the `cockroach-public` service. +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl label svc my-release-cockroachdb prometheus=cockroachdb + ~~~ + + ~~~ + service/my-release-cockroachdb labeled + ~~~ + + This ensures that there is a Prometheus job and monitoring data only for the `my-release-cockroachdb` service, not for the `my-release-cockroach-public` service. +
+ +2. Install [CoreOS's Prometheus Operator](https://raw.githubusercontent.com/coreos/prometheus-operator/release-0.20/bundle.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl apply \ + -f https://raw.githubusercontent.com/coreos/prometheus-operator/release-0.20/bundle.yaml + ~~~ + + ~~~ + clusterrolebinding.rbac.authorization.k8s.io/prometheus-operator created + clusterrole.rbac.authorization.k8s.io/prometheus-operator created + serviceaccount/prometheus-operator created + deployment.apps/prometheus-operator created + ~~~ +3. Confirm that the `prometheus-operator` has started: + + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get deploy prometheus-operator + ~~~ + + ~~~ + NAME READY UP-TO-DATE AVAILABLE AGE + prometheus-operator 1/1 1 1 27s + ~~~ + +4. Use our [`prometheus.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/prometheus/prometheus.yaml) file to create the various objects necessary to run a Prometheus instance: + + {{site.data.alerts.callout_success}} + This configuration defaults to using the Kubernetes CA for authentication. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl apply \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/prometheus/prometheus.yaml + ~~~ + + ~~~ + serviceaccount/prometheus created + clusterrole.rbac.authorization.k8s.io/prometheus created + clusterrolebinding.rbac.authorization.k8s.io/prometheus created + servicemonitor.monitoring.coreos.com/cockroachdb created + prometheus.monitoring.coreos.com/cockroachdb created + ~~~ + +5. Access the Prometheus UI locally and verify that CockroachDB is feeding data into Prometheus: + + 1. Port-forward from your local machine to the pod running Prometheus: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl port-forward prometheus-cockroachdb-0 9090 + ~~~ + + 2. Go to http://localhost:9090 in your browser. + + 3. To verify that each CockroachDB node is connected to Prometheus, go to **Status > Targets**. The screen should look like this: + + Prometheus targets + + 4. To verify that data is being collected, go to **Graph**, enter the `sys_uptime` variable in the field, click **Execute**, and then click the **Graph** tab. The screen should like this: + + Prometheus graph + + {{site.data.alerts.callout_success}} + Prometheus auto-completes CockroachDB time series metrics for you, but if you want to see a full listing, with descriptions, port-forward as described in {% if page.secure == true %}[Access the Admin UI](#step-4-access-the-admin-ui){% else %}[Access the Admin UI](#step-4-access-the-admin-ui){% endif %} and then point your browser to http://localhost:8080/_status/vars. + + For more details on using the Prometheus UI, see their [official documentation](https://prometheus.io/docs/introduction/getting_started/). + {{site.data.alerts.end}} + +### Configure Alertmanager + +Active monitoring helps you spot problems early, but it is also essential to send notifications when there are events that require investigation or intervention. This section shows you how to use [Alertmanager](https://prometheus.io/docs/alerting/alertmanager/) and CockroachDB's starter [alerting rules](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/prometheus/alert-rules.yaml) to do this. + +1. Download our alertmanager-config.yaml configuration file: + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -OOOOOOOOO \ + https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/prometheus/alertmanager-config.yaml + ~~~ + +2. Edit the `alertmanager-config.yaml` file to [specify the desired receivers for notifications](https://prometheus.io/docs/alerting/configuration/#receiver). Initially, the file contains a dummy web hook. + +3. Add this configuration to the Kubernetes cluster as a secret, renaming it to `alertmanager.yaml` and labelling it to make it easier to find: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create secret generic alertmanager-cockroachdb \ + --from-file=alertmanager.yaml=alertmanager-config.yaml + ~~~ + + ~~~ + secret/alertmanager-cockroachdb created + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl label secret alertmanager-cockroachdb app=cockroachdb + ~~~ + + ~~~ + secret/alertmanager-cockroachdb labeled + ~~~ + + {{site.data.alerts.callout_danger}} + The name of the secret, `alertmanager-cockroachdb`, must match the name used in the `alertmanager.yaml` file. If they differ, the Alertmanager instance will start without configuration, and nothing will happen. + {{site.data.alerts.end}} + +4. Use our [`alertmanager.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/prometheus/alertmanager.yaml) file to create the various objects necessary to run an Alertmanager instance, including a ClusterIP service so that Prometheus can forward alerts: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl apply \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/prometheus/alertmanager.yaml + ~~~ + + ~~~ + alertmanager.monitoring.coreos.com/cockroachdb created + service/alertmanager-cockroachdb created + ~~~ + +5. Verify that Alertmanager is running: + + 1. Port-forward from your local machine to the pod running Alertmanager: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl port-forward alertmanager-cockroachdb-0 9093 + ~~~ + + 2. Go to http://localhost:9093 in your browser. The screen should look like this: + + Alertmanager + +6. Ensure that the Alertmanagers are visible to Prometheus by opening http://localhost:9090/status. The screen should look like this: + + Alertmanager + +7. Add CockroachDB's starter [alerting rules](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/prometheus/alert-rules.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl apply \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/prometheus/alert-rules.yaml + ~~~ + + ~~~ + prometheusrule.monitoring.coreos.com/prometheus-cockroachdb-rules created + ~~~ + +8. Ensure that the rules are visible to Prometheus by opening http://localhost:9090/rules. The screen should look like this: + + Alertmanager + +9. Verify that the `TestAlertManager` example alert is firing by opening http://localhost:9090/alerts. The screen should look like this: + + Alertmanager + +10. To remove the example alert: + + 1. Use the `kubectl edit` command to open the rules for editing: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl edit prometheusrules prometheus-cockroachdb-rules + ~~~ + + 2. Remove the `dummy.rules` block and save the file: + + ~~~ + - name: rules/dummy.rules + rules: + - alert: TestAlertManager + expr: vector(1) + ~~~ diff --git a/_includes/v20.2/orchestration/kubernetes-remove-nodes-insecure.md b/_includes/v20.2/orchestration/kubernetes-remove-nodes-insecure.md new file mode 100644 index 00000000000..cf1eaeea910 --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-remove-nodes-insecure.md @@ -0,0 +1,130 @@ +To safely remove a node from your cluster, you must first decommission the node and only then adjust the `spec.replicas` value of your StatefulSet configuration to permanently remove it. This sequence is important because the decommissioning process lets a node finish in-flight requests, rejects any new requests, and transfers all range replicas and range leases off the node. + +{{site.data.alerts.callout_danger}} +If you remove nodes without first telling CockroachDB to decommission them, you may cause data or even cluster unavailability. For more details about how this works and what to consider before removing nodes, see [Decommission Nodes](remove-nodes.html). +{{site.data.alerts.end}} + +1. Launch a temporary interactive pod and use the `cockroach node status` command to get the internal IDs of nodes: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- node status \ + --insecure \ + --host=cockroachdb-public + ~~~ + + ~~~ + id | address | build | started_at | updated_at | is_available | is_live + +----+---------------------------------------------------------------------------------+--------+----------------------------------+----------------------------------+--------------+---------+ + 1 | cockroachdb-0.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:36.486082+00:00 | 2018-11-29 18:24:24.587454+00:00 | true | true + 2 | cockroachdb-2.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:55:03.880406+00:00 | 2018-11-29 18:24:23.469302+00:00 | true | true + 3 | cockroachdb-1.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:41.383588+00:00 | 2018-11-29 18:24:25.030175+00:00 | true | true + 4 | cockroachdb-3.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 17:31:19.990784+00:00 | 2018-11-29 18:24:26.041686+00:00 | true | true + (4 rows) + ~~~ + +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- node status \ + --insecure \ + --host=my-release-cockroachdb-public + ~~~ + + ~~~ + id | address | build | started_at | updated_at | is_available | is_live + +----+---------------------------------------------------------------------------------+--------+----------------------------------+----------------------------------+--------------+---------+ + 1 | my-release-cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:36.486082+00:00 | 2018-11-29 18:24:24.587454+00:00 | true | true + 2 | my-release-cockroachdb-2.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:55:03.880406+00:00 | 2018-11-29 18:24:23.469302+00:00 | true | true + 3 | my-release-cockroachdb-1.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:41.383588+00:00 | 2018-11-29 18:24:25.030175+00:00 | true | true + 4 | my-release-cockroachdb-3.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 17:31:19.990784+00:00 | 2018-11-29 18:24:26.041686+00:00 | true | true + (4 rows) + ~~~ +
+ +2. Note the ID of the node with the highest number in its address (in this case, the address including `cockroachdb-3`) and use the [`cockroach node decommission`](cockroach-node.html) command to decommission it: + + {{site.data.alerts.callout_info}} + It's important to decommission the node with the highest number in its address because, when you reduce the replica count, Kubernetes will remove the pod for that node. + {{site.data.alerts.end}} + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- node decommission \ + --insecure \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- node decommission \ + --insecure \ + --host=my-release-cockroachdb-public + ~~~ +
+ + You'll then see the decommissioning status print to `stderr` as it changes: + + ~~~ + id | is_live | replicas | is_decommissioning | is_draining + +---+---------+----------+--------------------+-------------+ + 4 | true | 73 | true | false + (1 row) + ~~~ + + Once the node has been fully decommissioned and stopped, you'll see a confirmation: + + ~~~ + id | is_live | replicas | is_decommissioning | is_draining + +---+---------+----------+--------------------+-------------+ + 4 | true | 0 | true | false + (1 row) + + No more data reported on target nodes. Please verify cluster health before removing the nodes. + ~~~ + +3. Once the node has been decommissioned, remove a pod from your StatefulSet: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl scale statefulset cockroachdb --replicas=3 + ~~~ + + ~~~ + statefulset "cockroachdb" scaled + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ helm upgrade \ + my-release \ + stable/cockroachdb \ + --set statefulset.replicas=3 \ + --reuse-values + ~~~ +
diff --git a/_includes/v20.2/orchestration/kubernetes-remove-nodes-secure.md b/_includes/v20.2/orchestration/kubernetes-remove-nodes-secure.md new file mode 100644 index 00000000000..3d6d1103e9f --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-remove-nodes-secure.md @@ -0,0 +1,119 @@ +To safely remove a node from your cluster, you must first decommission the node and only then adjust the `spec.replicas` value of your StatefulSet configuration to permanently remove it. This sequence is important because the decommissioning process lets a node finish in-flight requests, rejects any new requests, and transfers all range replicas and range leases off the node. + +{{site.data.alerts.callout_danger}} +If you remove nodes without first telling CockroachDB to decommission them, you may cause data or even cluster unavailability. For more details about how this works and what to consider before removing nodes, see [Decommission Nodes](remove-nodes.html). +{{site.data.alerts.end}} + +1. Get a shell into the `cockroachdb-client-secure` pod you created earlier and use the `cockroach node status` command to get the internal IDs of nodes: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach node status \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ + + ~~~ + id | address | build | started_at | updated_at | is_available | is_live + +----+---------------------------------------------------------------------------------+--------+----------------------------------+----------------------------------+--------------+---------+ + 1 | cockroachdb-0.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:36.486082+00:00 | 2018-11-29 18:24:24.587454+00:00 | true | true + 2 | cockroachdb-2.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:55:03.880406+00:00 | 2018-11-29 18:24:23.469302+00:00 | true | true + 3 | cockroachdb-1.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:41.383588+00:00 | 2018-11-29 18:24:25.030175+00:00 | true | true + 4 | cockroachdb-3.cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 17:31:19.990784+00:00 | 2018-11-29 18:24:26.041686+00:00 | true | true + (4 rows) + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach node status \ + --certs-dir=/cockroach-certs \ + --host=my-release-cockroachdb-public + ~~~ + + ~~~ + id | address | build | started_at | updated_at | is_available | is_live + +----+---------------------------------------------------------------------------------+--------+----------------------------------+----------------------------------+--------------+---------+ + 1 | my-release-cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:36.486082+00:00 | 2018-11-29 18:24:24.587454+00:00 | true | true + 2 | my-release-cockroachdb-2.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:55:03.880406+00:00 | 2018-11-29 18:24:23.469302+00:00 | true | true + 3 | my-release-cockroachdb-1.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 16:04:41.383588+00:00 | 2018-11-29 18:24:25.030175+00:00 | true | true + 4 | my-release-cockroachdb-3.my-release-cockroachdb.default.svc.cluster.local:26257 | {{page.release_info.version}} | 2018-11-29 17:31:19.990784+00:00 | 2018-11-29 18:24:26.041686+00:00 | true | true + (4 rows) + ~~~ +
+ + The pod uses the `root` client certificate created earlier to initialize the cluster, so there's no CSR approval required. + +2. Note the ID of the node with the highest number in its address (in this case, the address including `cockroachdb-3`) and use the [`cockroach node decommission`](cockroach-node.html) command to decommission it: + + {{site.data.alerts.callout_info}} + It's important to decommission the node with the highest number in its address because, when you reduce the replica count, Kubernetes will remove the pod for that node. + {{site.data.alerts.end}} + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach node decommission \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach node decommission \ + --certs-dir=/cockroach-certs \ + --host=my-release-cockroachdb-public + ~~~ +
+ + You'll then see the decommissioning status print to `stderr` as it changes: + + ~~~ + id | is_live | replicas | is_decommissioning | is_draining + +---+---------+----------+--------------------+-------------+ + 4 | true | 73 | true | false + (1 row) + ~~~ + + Once the node has been fully decommissioned and stopped, you'll see a confirmation: + + ~~~ + id | is_live | replicas | is_decommissioning | is_draining + +---+---------+----------+--------------------+-------------+ + 4 | true | 0 | true | false + (1 row) + + No more data reported on target nodes. Please verify cluster health before removing the nodes. + ~~~ + +3. Once the node has been decommissioned, remove a pod from your StatefulSet: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl scale statefulset cockroachdb --replicas=3 + ~~~ + + ~~~ + statefulset.apps/cockroachdb scaled + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ helm upgrade \ + my-release \ + stable/cockroachdb \ + --set statefulset.replicas=3 \ + --reuse-values + ~~~ +
diff --git a/_includes/v20.2/orchestration/kubernetes-scale-cluster.md b/_includes/v20.2/orchestration/kubernetes-scale-cluster.md new file mode 100644 index 00000000000..8eb49b7d14f --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-scale-cluster.md @@ -0,0 +1,61 @@ +Your Kubernetes cluster includes 3 worker nodes, or instances, that can run pods. A CockroachDB node runs in each pod. As recommended in our [production best practices](recommended-production-settings.html#topology), you should ensure that two pods are not placed on the same worker node. + + +To do this, add a new worker node and then edit your StatefulSet configuration to add another pod for the new CockroachDB node. + +1. Add a worker node, bringing the total from 3 to 4: + - On GKE, [resize your cluster](https://cloud.google.com/kubernetes-engine/docs/how-to/resizing-a-cluster). + - On EKS, resize your [Worker Node Group](https://eksctl.io/usage/managing-nodegroups/#scaling). + - On GCE, resize your [Managed Instance Group](https://cloud.google.com/compute/docs/instance-groups/). + - On AWS, resize your [Auto Scaling Group](https://docs.aws.amazon.com/autoscaling/latest/userguide/as-manual-scaling.html). + +2. Add a pod for the new CockroachDB node: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl scale statefulset cockroachdb --replicas=4 + ~~~ + + ~~~ + statefulset.apps/cockroachdb scaled + ~~~ + + {{site.data.alerts.callout_success}} + If you aren't using the Kubernetes CA to sign certificates, you can now skip to step 6. + {{site.data.alerts.end}} +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ helm upgrade \ + my-release \ + stable/cockroachdb \ + --set statefulset.replicas=4 \ + --reuse-values + ~~~ + + ~~~ + Release "my-release" has been upgraded. Happy Helming! + LAST DEPLOYED: Tue May 14 14:06:43 2019 + NAMESPACE: default + STATUS: DEPLOYED + + RESOURCES: + ==> v1beta1/PodDisruptionBudget + NAME AGE + my-release-cockroachdb-budget 51m + + ==> v1/Pod(related) + + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 38m + my-release-cockroachdb-1 1/1 Running 0 39m + my-release-cockroachdb-2 1/1 Running 0 39m + my-release-cockroachdb-3 0/1 Pending 0 0s + my-release-cockroachdb-init-nwjkh 0/1 Completed 0 39m + + ... + ~~~ +
diff --git a/_includes/v20.2/orchestration/kubernetes-simulate-failure.md b/_includes/v20.2/orchestration/kubernetes-simulate-failure.md new file mode 100644 index 00000000000..d71bd76bfaa --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-simulate-failure.md @@ -0,0 +1,56 @@ +Based on the `replicas: 3` line in the StatefulSet configuration, Kubernetes ensures that three pods/nodes are running at all times. When a pod/node fails, Kubernetes automatically creates another pod/node with the same network identity and persistent storage. + +To see this in action: + +1. Kill one of CockroachDB nodes: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete pod cockroachdb-2 + ~~~ + + ~~~ + pod "cockroachdb-2" deleted + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete pod my-release-cockroachdb-2 + ~~~ + + ~~~ + pod "my-release-cockroachdb-2" deleted + ~~~ +
+ + +2. In the Admin UI, the **Cluster Overview** will soon show one node as **Suspect**. As Kubernetes auto-restarts the node, watch how the node once again becomes healthy. + +3. Back in the terminal, verify that the pod was automatically restarted: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pod cockroachdb-2 + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-2 1/1 Running 0 12s + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pod my-release-cockroachdb-2 + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-2 1/1 Running 0 44s + ~~~ +
diff --git a/_includes/v20.2/orchestration/kubernetes-upgrade-cluster.md b/_includes/v20.2/orchestration/kubernetes-upgrade-cluster.md new file mode 100644 index 00000000000..14c6e0e2010 --- /dev/null +++ b/_includes/v20.2/orchestration/kubernetes-upgrade-cluster.md @@ -0,0 +1,265 @@ +As new versions of CockroachDB are released, it's strongly recommended to upgrade to newer versions in order to pick up bug fixes, performance improvements, and new features. The [general CockroachDB upgrade documentation](upgrade-cockroach-version.html) provides best practices for how to prepare for and execute upgrades of CockroachDB clusters, but the mechanism of actually stopping and restarting processes in Kubernetes is somewhat special. + +Kubernetes knows how to carry out a safe rolling upgrade process of the CockroachDB nodes. When you tell it to change the Docker image used in the CockroachDB StatefulSet, Kubernetes will go one-by-one, stopping a node, restarting it with the new image, and waiting for it to be ready to receive client requests before moving on to the next one. For more information, see [the Kubernetes documentation](https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets). + +1. Decide how the upgrade will be finalized. + + {{site.data.alerts.callout_info}} + This step is relevant only when upgrading from v20.1.x to v20.2. For upgrades within the v20.2.x series, skip this step. + {{site.data.alerts.end}} + + By default, after all nodes are running the new version, the upgrade process will be **auto-finalized**. This will enable certain performance improvements and bug fixes introduced in v20.2. After finalization, however, it will no longer be possible to perform a downgrade to v20.1. In the event of a catastrophic failure or corruption, the only option will be to start a new cluster using the old binary and then restore from one of the backups created prior to performing the upgrade. + + We recommend disabling auto-finalization so you can monitor the stability and performance of the upgraded cluster before finalizing the upgrade: + + {% if page.secure == true %} + + 1. Get a shell into the pod with the `cockroach` binary created earlier and start the CockroachDB [built-in SQL client](cockroach-sql.html): + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \-- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=my-release-cockroachdb-public + ~~~ +
+ + + {% else %} + + 1. Launch a temporary interactive pod and start the [built-in SQL client](cockroach-sql.html) inside it: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=my-release-cockroachdb-public + ~~~ +
+ + {% endif %} + + 2. Set the `cluster.preserve_downgrade_option` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING cluster.preserve_downgrade_option = '19.2'; + ~~~ + + 3. Exit the SQL shell and delete the temporary pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +2. Kick off the upgrade process by changing the desired Docker image: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl patch statefulset cockroachdb \ + --type='json' \ + -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value":"cockroachdb/cockroach:{{page.release_info.version}}"}]' + ~~~ + + ~~~ + statefulset.apps/cockroachdb patched + ~~~ +
+ +
+ + {{site.data.alerts.callout_info}} + For Helm, you must remove the cluster initialization job from when the cluster was created before the cluster version can be changed. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete job my-release-cockroachdb-init + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ helm upgrade \ + my-release \ + stable/cockroachdb \ + --set image.tag={{page.release_info.version}} \ + --reuse-values + ~~~ +
+ +3. If you then check the status of your cluster's pods, you should see them being restarted: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + +
+ ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 1/1 Running 0 2m + cockroachdb-1 1/1 Running 0 2m + cockroachdb-2 1/1 Running 0 2m + cockroachdb-3 0/1 Terminating 0 1m + ... + ~~~ +
+ +
+ ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 2m + my-release-cockroachdb-1 1/1 Running 0 3m + my-release-cockroachdb-2 1/1 Running 0 3m + my-release-cockroachdb-3 0/1 ContainerCreating 0 25s + my-release-cockroachdb-init-nwjkh 0/1 ContainerCreating 0 6s + ... + ~~~ + + {{site.data.alerts.callout_info}} + Ignore the pod for cluster initialization. It is re-created as a byproduct of the StatefulSet configuration but does not impact your existing cluster. + {{site.data.alerts.end}} +
+ +4. This will continue until all of the pods have restarted and are running the new image. To check the image of each pod to determine whether they've all be upgraded, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.spec.containers[0].image}{"\n"}' + ~~~ + +
+ ~~~ + cockroachdb-0 cockroachdb/cockroach:{{page.release_info.version}} + cockroachdb-1 cockroachdb/cockroach:{{page.release_info.version}} + cockroachdb-2 cockroachdb/cockroach:{{page.release_info.version}} + cockroachdb-3 cockroachdb/cockroach:{{page.release_info.version}} + ... + ~~~ +
+ +
+ ~~~ + my-release-cockroachdb-0 cockroachdb/cockroach:{{page.release_info.version}} + my-release-cockroachdb-1 cockroachdb/cockroach:{{page.release_info.version}} + my-release-cockroachdb-2 cockroachdb/cockroach:{{page.release_info.version}} + my-release-cockroachdb-3 cockroachdb/cockroach:{{page.release_info.version}} + ... + ~~~ +
+ + You can also check the CockroachDB version of each node in the Admin UI: + + Version in UI after upgrade + +5. Finish the upgrade. + + {{site.data.alerts.callout_info}} + This step is relevant only when upgrading from v20.1.x to v20.2. For upgrades within the v20.2.x series, skip this step. + {{site.data.alerts.end}} + + If you disabled auto-finalization in step 1 above, monitor the stability and performance of your cluster for as long as you require to feel comfortable with the upgrade (generally at least a day). If during this time you decide to roll back the upgrade, repeat the rolling restart procedure with the old binary. + + Once you are satisfied with the new version, re-enable auto-finalization: + + {% if page.secure == true %} + + 1. Get a shell into the pod with the `cockroach` binary created earlier and start the CockroachDB [built-in SQL client](cockroach-sql.html): + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=my-release-cockroachdb-public + ~~~ +
+ + {% else %} + + 1. Launch a temporary interactive pod and start the [built-in SQL client](cockroach-sql.html) inside it: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=my-release-cockroachdb-public + ~~~ +
+ + {% endif %} + + 2. Re-enable auto-finalization: + + {% include copy-clipboard.html %} + ~~~ sql + > RESET CLUSTER SETTING cluster.preserve_downgrade_option; + ~~~ + + 3. Exit the SQL shell and delete the temporary pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ diff --git a/_includes/v20.2/orchestration/local-start-kubernetes.md b/_includes/v20.2/orchestration/local-start-kubernetes.md new file mode 100644 index 00000000000..081c2274c0f --- /dev/null +++ b/_includes/v20.2/orchestration/local-start-kubernetes.md @@ -0,0 +1,24 @@ +## Before you begin + +Before getting started, it's helpful to review some Kubernetes-specific terminology: + +Feature | Description +--------|------------ +[minikube](http://kubernetes.io/docs/getting-started-guides/minikube/) | This is the tool you'll use to run a Kubernetes cluster inside a VM on your local workstation. +[pod](http://kubernetes.io/docs/user-guide/pods/) | A pod is a group of one of more Docker containers. In this tutorial, all pods will run on your local workstation, each containing one Docker container running a single CockroachDB node. You'll start with 3 pods and grow to 4. +[StatefulSet](http://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/) | A StatefulSet is a group of pods treated as stateful units, where each pod has distinguishable network identity and always binds back to the same persistent storage on restart. StatefulSets are considered stable as of Kubernetes version 1.9 after reaching beta in version 1.5. +[persistent volume](http://kubernetes.io/docs/user-guide/persistent-volumes/) | A persistent volume is a piece of storage mounted into a pod. The lifetime of a persistent volume is decoupled from the lifetime of the pod that's using it, ensuring that each CockroachDB node binds back to the same storage on restart.

When using `minikube`, persistent volumes are external temporary directories that endure until they are manually deleted or until the entire Kubernetes cluster is deleted. +[persistent volume claim](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims) | When pods are created (one per CockroachDB node), each pod will request a persistent volume claim to “claim” durable storage for its node. + +## Step 1. Start Kubernetes + +1. Follow Kubernetes' [documentation](https://kubernetes.io/docs/tasks/tools/install-minikube/) to install `minikube`, the tool used to run Kubernetes locally, for your OS. This includes installing a hypervisor and `kubectl`, the command-line tool used to manage Kubernetes from your local workstation. + + {{site.data.alerts.callout_info}}Make sure you install minikube version 0.21.0 or later. Earlier versions do not include a Kubernetes server that supports the maxUnavailability field and PodDisruptionBudget resource type used in the CockroachDB StatefulSet configuration.{{site.data.alerts.end}} + +2. Start a local Kubernetes cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ minikube start + ~~~ diff --git a/_includes/v20.2/orchestration/monitor-cluster.md b/_includes/v20.2/orchestration/monitor-cluster.md new file mode 100644 index 00000000000..16914b30ca3 --- /dev/null +++ b/_includes/v20.2/orchestration/monitor-cluster.md @@ -0,0 +1,69 @@ +To access the cluster's [Admin UI](admin-ui-overview.html): + +{% if page.secure == true %} + +1. On secure clusters, [certain pages of the Admin UI](admin-ui-overview.html#admin-ui-access) can only be accessed by `admin` users. + + Get a shell into the pod and start the CockroachDB [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ + +1. Assign `roach` to the `admin` role (you only need to do this once): + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT admin TO roach; + ~~~ + +1. Exit the SQL shell and pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +{% endif %} + +1. In a new terminal window, port-forward from your local machine to one of the pods: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl port-forward cockroachdb-0 8080 + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl port-forward my-release-cockroachdb-0 8080 + ~~~ +
+ + ~~~ + Forwarding from 127.0.0.1:8080 -> 8080 + ~~~ + + {{site.data.alerts.callout_info}}The port-forward command must be run on the same machine as the web browser in which you want to view the Admin UI. If you have been running these commands from a cloud instance or other non-local shell, you will not be able to view the UI without configuring kubectl locally and running the above port-forward command on your local machine.{{site.data.alerts.end}} + +{% if page.secure == true %} + +1. Go to https://localhost:8080 and log in with the username and password you created earlier. + + {% include {{ page.version.version }}/misc/chrome-localhost.md %} + +{% else %} + +1. Go to http://localhost:8080. + +{% endif %} + +1. In the UI, verify that the cluster is running as expected: + - Click **View nodes list** on the right to ensure that all nodes successfully joined the cluster. + - Click the **Databases** tab on the left to verify that `bank` is listed. diff --git a/_includes/v20.2/orchestration/start-cockroachdb-helm-insecure.md b/_includes/v20.2/orchestration/start-cockroachdb-helm-insecure.md new file mode 100644 index 00000000000..35ca5d4ea70 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-helm-insecure.md @@ -0,0 +1,94 @@ +1. [Install the Helm client](https://helm.sh/docs/intro/install) (version 3.0 or higher) and add the official `stable` chart repository: + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo add stable https://kubernetes-charts.storage.googleapis.com + ~~~ + + ~~~ + "stable" has been added to your repositories + ~~~ + +2. Update your Helm chart repositories to ensure that you're using the [latest CockroachDB chart](https://github.com/helm/charts/blob/master/stable/cockroachdb/Chart.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo update + ~~~ + +3. Modify our Helm chart's [`values.yaml`](https://github.com/helm/charts/blob/master/stable/cockroachdb/values.yaml) parameters for your deployment scenario. + + Create a `my-values.yaml` file to override the defaults in `values.yaml`, substituting your own values in this example based on the guidelines below. + + {% include copy-clipboard.html %} + ~~~ + statefulset: + resources: + limits: + memory: "8Gi" + requests: + memory: "8Gi" + conf: + cache: "2Gi" + max-sql-memory: "2Gi" + ~~~ + + 1. To avoid running out of memory when CockroachDB is not the only pod on a Kubernetes node, you *must* set memory limits explicitly. This is because CockroachDB does not detect the amount of memory allocated to its pod when run in Kubernetes. We recommend setting `conf.cache` and `conf.max-sql-memory` each to 1/4 of the `memory` allocation specified in `statefulset.resources.requests` and `statefulset.resources.limits`. + + {{site.data.alerts.callout_success}} + For example, if you are allocating 8Gi of `memory` to each CockroachDB node, allocate 2Gi to `cache` and 2Gi to `max-sql-memory`. + {{site.data.alerts.end}} + + 2. You may want to modify `storage.persistentVolume.size` and `storage.persistentVolume.storageClass` for your use case. This chart defaults to 100Gi of disk space per pod. For more details on customizing disks for performance, see [these instructions](kubernetes-performance.html#disk-type). + + {{site.data.alerts.callout_info}} + If necessary, you can [expand disk size](orchestrate-cockroachdb-with-kubernetes.html#expand-disk-size) after the cluster is live. + {{site.data.alerts.end}} + +4. Install the CockroachDB Helm chart. + + Provide a "release" name to identify and track this particular deployment of the chart, and override the default values with those in `my-values.yaml`. + + {{site.data.alerts.callout_info}} + This tutorial uses `my-release` as the release name. If you use a different value, be sure to adjust the release name in subsequent commands. Also be sure to start and end the name with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ helm install my-release --values my-values.yaml stable/cockroachdb + ~~~ + + Behind the scenes, this command uses our `cockroachdb-statefulset.yaml` file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it, where each pod has distinguishable network identity and always binds back to the same persistent storage on restart. + +5. Confirm that CockroachDB cluster initialization has completed successfully, with the pods for CockroachDB showing `1/1` under `READY` and the pod for initialization showing `COMPLETED` under `STATUS`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 8m + my-release-cockroachdb-1 1/1 Running 0 8m + my-release-cockroachdb-2 1/1 Running 0 8m + my-release-cockroachdb-init-hxzsc 0/1 Completed 0 1h + ~~~ + +6. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-71019b3a-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 11m + pvc-7108e172-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 11m + pvc-710dcb66-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 11m + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-helm-secure.md b/_includes/v20.2/orchestration/start-cockroachdb-helm-secure.md new file mode 100644 index 00000000000..dd10c4c5d19 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-helm-secure.md @@ -0,0 +1,185 @@ +1. [Install the Helm client](https://helm.sh/docs/intro/install) (version 3.0 or higher) and add the official `stable` chart repository: + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo add stable https://kubernetes-charts.storage.googleapis.com + ~~~ + + ~~~ + "stable" has been added to your repositories + ~~~ + +2. Update your Helm chart repositories to ensure that you're using the [latest CockroachDB chart](https://github.com/helm/charts/blob/master/stable/cockroachdb/Chart.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo update + ~~~ + +3. Modify our Helm chart's [`values.yaml`](https://github.com/helm/charts/blob/master/stable/cockroachdb/values.yaml) parameters for your deployment scenario. + + Create a `my-values.yaml` file to override the defaults in `values.yaml`, substituting your own values in this example based on the guidelines below. + + {% include copy-clipboard.html %} + ~~~ + statefulset: + resources: + limits: + memory: "8Gi" + requests: + memory: "8Gi" + conf: + cache: "2Gi" + max-sql-memory: "2Gi" + tls: + enabled: true + ~~~ + + 1. To avoid running out of memory when CockroachDB is not the only pod on a Kubernetes node, you *must* set memory limits explicitly. This is because CockroachDB does not detect the amount of memory allocated to its pod when run in Kubernetes. We recommend setting `conf.cache` and `conf.max-sql-memory` each to 1/4 of the `memory` allocation specified in `statefulset.resources.requests` and `statefulset.resources.limits`. + + {{site.data.alerts.callout_success}} + For example, if you are allocating 8Gi of `memory` to each CockroachDB node, allocate 2Gi to `cache` and 2Gi to `max-sql-memory`. + {{site.data.alerts.end}} + + 2. You may want to modify `storage.persistentVolume.size` and `storage.persistentVolume.storageClass` for your use case. This chart defaults to 100Gi of disk space per pod. For more details on customizing disks for performance, see [these instructions](kubernetes-performance.html#disk-type). + + {{site.data.alerts.callout_info}} + If necessary, you can [expand disk size](orchestrate-cockroachdb-with-kubernetes.html#expand-disk-size) after the cluster is live. + {{site.data.alerts.end}} + + 3. For a secure deployment, set `tls.enabled` to true. + +4. Install the CockroachDB Helm chart. + + Provide a "release" name to identify and track this particular deployment of the chart, and override the default values with those in `my-values.yaml`. + + {{site.data.alerts.callout_info}} + This tutorial uses `my-release` as the release name. If you use a different value, be sure to adjust the release name in subsequent commands. Also be sure to start and end the name with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ helm install my-release --values my-values.yaml stable/cockroachdb + ~~~ + + Behind the scenes, this command uses our `cockroachdb-statefulset.yaml` file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it, where each pod has distinguishable network identity and always binds back to the same persistent storage on restart. + +6. As each pod is created, it issues a Certificate Signing Request, or CSR, to have the CockroachDB node's certificate signed by the Kubernetes CA. You must manually check and approve each node's certificate, at which point the CockroachDB node is started in the pod. + + 1. Get the names of the `Pending` CSRs: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get csr + ~~~ + + ~~~ + NAME AGE REQUESTOR CONDITION + default.client.root 21s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-0 15s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-1 16s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-2 15s system:serviceaccount:default:my-release-cockroachdb Pending + ... + ~~~ + + If you do not see a `Pending` CSR, wait a minute and try again. + + 2. Examine the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe csr default.node.my-release-cockroachdb-0 + ~~~ + + ~~~ + Name: default.node.my-release-cockroachdb-0 + Labels: + Annotations: + CreationTimestamp: Mon, 10 Dec 2018 05:36:35 -0500 + Requesting User: system:serviceaccount:default:my-release-cockroachdb + Status: Pending + Subject: + Common Name: node + Serial Number: + Organization: Cockroach + Subject Alternative Names: + DNS Names: localhost + my-release-cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local + my-release-cockroachdb-0.my-release-cockroachdb + my-release-cockroachdb-public + my-release-cockroachdb-public.default.svc.cluster.local + IP Addresses: 127.0.0.1 + Events: + ~~~ + + 3. If everything looks correct, approve the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.node.my-release-cockroachdb-0 + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.node.my-release-cockroachdb-0 approved + ~~~ + + 4. Repeat steps 2 and 3 for the other 2 pods. + +7. Confirm that three pods are `Running` successfully: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 0/1 Running 0 6m + my-release-cockroachdb-1 0/1 Running 0 6m + my-release-cockroachdb-2 0/1 Running 0 6m + my-release-cockroachdb-init-hxzsc 0/1 Init:0/1 0 6m + ~~~ + +8. Approve the CSR for the one-off pod from which cluster initialization happens: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.client.root + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.client.root approved + ~~~ + +9. Confirm that CockroachDB cluster initialization has completed successfully, with the pods for CockroachDB showing `1/1` under `READY` and the pod for initialization showing `COMPLETED` under `STATUS`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 8m + my-release-cockroachdb-1 1/1 Running 0 8m + my-release-cockroachdb-2 1/1 Running 0 8m + my-release-cockroachdb-init-hxzsc 0/1 Completed 0 1h + ~~~ + +10. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-71019b3a-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 11m + pvc-7108e172-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 11m + pvc-710dcb66-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 11m + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-insecure.md b/_includes/v20.2/orchestration/start-cockroachdb-insecure.md new file mode 100644 index 00000000000..cb3910c0fb0 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-insecure.md @@ -0,0 +1,121 @@ +1. From your local workstation, use our [`cockroachdb-statefulset.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset.yaml) file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it. + + Download [`cockroachdb-statefulset.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cockroachdb-statefulset.yaml + ~~~ + + {{site.data.alerts.callout_danger}} + To avoid running out of memory when CockroachDB is not the only pod on a Kubernetes node, you must set memory limits explicitly. This is because CockroachDB does not detect the amount of memory allocated to its pod when run in Kubernetes. Specify this amount by adjusting `resources.requests.memory` and `resources.limits.memory` in `cockroachdb-statefulset.yaml`. Their values should be identical. + + We recommend setting `cache` and `max-sql-memory` each to 1/4 of your memory allocation. For example, if you are allocating 8Gi of memory to each CockroachDB node, substitute the following values in [this line](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset.yaml#L146): + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + --cache 2Gi --max-sql-memory 2Gi + ~~~ + + Use the file to create the StatefulSet and start the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset.yaml + ~~~ + + ~~~ + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + + Alternatively, if you'd rather start with a configuration file that has been customized for performance: + + 1. Download our [performance version of `cockroachdb-statefulset-insecure.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/performance/cockroachdb-statefulset-insecure.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/performance/cockroachdb-statefulset-insecure.yaml + ~~~ + + 2. Modify the file wherever there is a `TODO` comment. + + 3. Use the file to create the StatefulSet and start the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset-insecure.yaml + ~~~ + +2. Confirm that three pods are `Running` successfully. Note that they will not + be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + +3. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get persistentvolumes + ~~~ + + ~~~ + NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE + pvc-52f51ecf-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-0 26s + pvc-52fd3a39-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-1 27s + pvc-5315efda-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-2 27s + ~~~ + +4. Use our [`cluster-init.yaml`](https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init.yaml) file to perform a one-time initialization that joins the CockroachDB nodes into a single cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init.yaml + ~~~ + + ~~~ + job.batch/cluster-init created + ~~~ + +5. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get job cluster-init + ~~~ + + ~~~ + NAME COMPLETIONS DURATION AGE + cluster-init 1/1 7s 27s + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cluster-init-cqf8l 0/1 Completed 0 56s + cockroachdb-0 1/1 Running 0 7m51s + cockroachdb-1 1/1 Running 0 7m51s + cockroachdb-2 1/1 Running 0 7m51s + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-local-helm-insecure.md b/_includes/v20.2/orchestration/start-cockroachdb-local-helm-insecure.md new file mode 100644 index 00000000000..1aa46194329 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-local-helm-insecure.md @@ -0,0 +1,65 @@ +1. [Install the Helm client](https://helm.sh/docs/intro/install) and add the official `stable` chart repository: + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo add stable https://kubernetes-charts.storage.googleapis.com + ~~~ + + ~~~ + "stable" has been added to your repositories + ~~~ + +2. Update your Helm chart repositories to ensure that you're using the [latest CockroachDB chart](https://github.com/helm/charts/blob/master/stable/cockroachdb/Chart.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo update + ~~~ + +3. Install the CockroachDB Helm chart. + + Provide a "release" name to identify and track this particular deployment of the chart. + + {{site.data.alerts.callout_info}} + This tutorial uses `my-release` as the release name. If you use a different value, be sure to adjust the release name in subsequent commands. Also be sure to start and end the name with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ helm install my-release stable/cockroachdb + ~~~ + + Behind the scenes, this command uses our `cockroachdb-statefulset.yaml` file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it, where each pod has distinguishable network identity and always binds back to the same persistent storage on restart. + +4. Confirm that CockroachDB cluster initialization has completed successfully, with the pods for CockroachDB showing `1/1` under `READY` and the pod for initialization showing `COMPLETED` under `STATUS`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 8m + my-release-cockroachdb-1 1/1 Running 0 8m + my-release-cockroachdb-2 1/1 Running 0 8m + my-release-cockroachdb-init-hxzsc 0/1 Completed 0 1h + ~~~ + +5. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-71019b3a-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 11m + pvc-7108e172-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 11m + pvc-710dcb66-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 11m + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-local-helm-secure.md b/_includes/v20.2/orchestration/start-cockroachdb-local-helm-secure.md new file mode 100644 index 00000000000..bc11de7d52f --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-local-helm-secure.md @@ -0,0 +1,162 @@ +1. [Install the Helm client](https://helm.sh/docs/intro/install) and add the official `stable` chart repository: + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo add stable https://kubernetes-charts.storage.googleapis.com + ~~~ + + ~~~ + "stable" has been added to your repositories + ~~~ + +2. Update your Helm chart repositories to ensure that you're using the [latest CockroachDB chart](https://github.com/helm/charts/blob/master/stable/cockroachdb/Chart.yaml): + + {% include copy-clipboard.html %} + ~~~ shell + $ helm repo update + ~~~ + +3. Modify our Helm chart's [`values.yaml`](https://github.com/helm/charts/blob/master/stable/cockroachdb/values.yaml) parameters for your deployment scenario. + + Create a `my-values.yaml` file to override the defaults. For a secure deployment, set `tls.enabled` to true: + + {% include copy-clipboard.html %} + ~~~ + tls: + enabled: true + ~~~ + +4. Install the CockroachDB Helm chart. + + Provide a "release" name to identify and track this particular deployment of the chart, and override the default values with those in `my-values.yaml`. + + {{site.data.alerts.callout_info}} + This tutorial uses `my-release` as the release name. If you use a different value, be sure to adjust the release name in subsequent commands. Also be sure to start and end the name with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ helm install my-release --values my-values.yaml stable/cockroachdb + ~~~ + + Behind the scenes, this command uses our `cockroachdb-statefulset.yaml` file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it, where each pod has distinguishable network identity and always binds back to the same persistent storage on restart. + +6. As each pod is created, it issues a Certificate Signing Request, or CSR, to have the CockroachDB node's certificate signed by the Kubernetes CA. You must manually check and approve each node's certificate, at which point the CockroachDB node is started in the pod. + + 1. Get the names of the `Pending` CSRs: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get csr + ~~~ + + ~~~ + NAME AGE REQUESTOR CONDITION + default.client.root 21s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-0 15s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-1 16s system:serviceaccount:default:my-release-cockroachdb Pending + default.node.my-release-cockroachdb-2 15s system:serviceaccount:default:my-release-cockroachdb Pending + ... + ~~~ + + If you do not see a `Pending` CSR, wait a minute and try again. + + 2. Examine the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe csr default.node.my-release-cockroachdb-0 + ~~~ + + ~~~ + Name: default.node.my-release-cockroachdb-0 + Labels: + Annotations: + CreationTimestamp: Mon, 10 Dec 2018 05:36:35 -0500 + Requesting User: system:serviceaccount:default:my-release-cockroachdb + Status: Pending + Subject: + Common Name: node + Serial Number: + Organization: Cockroach + Subject Alternative Names: + DNS Names: localhost + my-release-cockroachdb-0.my-release-cockroachdb.default.svc.cluster.local + my-release-cockroachdb-0.my-release-cockroachdb + my-release-cockroachdb-public + my-release-cockroachdb-public.default.svc.cluster.local + IP Addresses: 127.0.0.1 + Events: + ~~~ + + 3. If everything looks correct, approve the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.node.my-release-cockroachdb-0 + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.node.my-release-cockroachdb-0 approved + ~~~ + + 4. Repeat steps 2 and 3 for the other 2 pods. + +7. Confirm that three pods are `Running` successfully: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 0/1 Running 0 6m + my-release-cockroachdb-1 0/1 Running 0 6m + my-release-cockroachdb-2 0/1 Running 0 6m + my-release-cockroachdb-init-hxzsc 0/1 Init:0/1 0 6m + ~~~ + +8. Approve the CSR for the one-off pod from which cluster initialization happens: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.client.root + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.client.root approved + ~~~ + +9. Confirm that CockroachDB cluster initialization has completed successfully, with the pods for CockroachDB showing `1/1` under `READY` and the pod for initialization showing `COMPLETED` under `STATUS`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + my-release-cockroachdb-0 1/1 Running 0 8m + my-release-cockroachdb-1 1/1 Running 0 8m + my-release-cockroachdb-2 1/1 Running 0 8m + my-release-cockroachdb-init-hxzsc 0/1 Completed 0 1h + ~~~ + +10. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-71019b3a-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-0 standard 11m + pvc-7108e172-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-1 standard 11m + pvc-710dcb66-fc67-11e8-a606-080027ba45e5 100Gi RWO Delete Bound default/datadir-my-release-cockroachdb-2 standard 11m + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-local-insecure.md b/_includes/v20.2/orchestration/start-cockroachdb-local-insecure.md new file mode 100644 index 00000000000..bebb6eb3062 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-local-insecure.md @@ -0,0 +1,83 @@ +1. From your local workstation, use our [`cockroachdb-statefulset.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset.yaml) file to create the StatefulSet that automatically creates 3 pods, each with a CockroachDB node running inside it: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cockroachdb-statefulset.yaml + ~~~ + + ~~~ + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + +2. Confirm that three pods are `Running` successfully. Note that they will not + be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + +3. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE + pvc-52f51ecf-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-0 26s + pvc-52fd3a39-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-1 27s + pvc-5315efda-8bd5-11e6-a4f4-42010a800002 1Gi RWO Delete Bound default/datadir-cockroachdb-2 27s + ~~~ + +4. Use our [`cluster-init.yaml`](https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init.yaml) file to perform a one-time initialization that joins the CockroachDB nodes into a single cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init.yaml + ~~~ + + ~~~ + job.batch/cluster-init created + ~~~ + +5. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get job cluster-init + ~~~ + + ~~~ + NAME COMPLETIONS DURATION AGE + cluster-init 1/1 7s 27s + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cluster-init-cqf8l 0/1 Completed 0 56s + cockroachdb-0 1/1 Running 0 7m51s + cockroachdb-1 1/1 Running 0 7m51s + cockroachdb-2 1/1 Running 0 7m51s + ~~~ + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/orchestration/start-cockroachdb-local-secure.md b/_includes/v20.2/orchestration/start-cockroachdb-local-secure.md new file mode 100644 index 00000000000..0558bcdf3b2 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-local-secure.md @@ -0,0 +1,366 @@ +Download and modify our StatefulSet configuration, depending on how you want to sign your certificates. + +{{site.data.alerts.callout_danger}} +Some environments, such as Amazon EKS, do not support certificates signed by Kubernetes' built-in CA. In this case, use the second configuration below. +{{site.data.alerts.end}} + +- Using the Kubernetes CA: [`cockroachdb-statefulset-secure.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset-secure.yaml). + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cockroachdb-statefulset-secure.yaml + ~~~ + +- Using a non-Kubernetes CA: [`cockroachdb-statefulset.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml) + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml + ~~~ + +{{site.data.alerts.callout_success}} +If you change the StatefulSet name from the default `cockroachdb`, be sure to start and end with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). +{{site.data.alerts.end}} + +#### Initialize the cluster + +Choose the authentication method that corresponds to the StatefulSet configuration you downloaded and modified above. + +- [Kubernetes CA](#kubernetes-ca) +- [Non-Kubernetes CA](#non-kubernetes-ca) + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} + +##### Kubernetes CA + +1. Use the config file you downloaded to create the StatefulSet that automatically creates 3 pods, each running a CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset-secure.yaml + ~~~ + + ~~~ + serviceaccount/cockroachdb created + role.rbac.authorization.k8s.io/cockroachdb created + clusterrole.rbac.authorization.k8s.io/cockroachdb created + rolebinding.rbac.authorization.k8s.io/cockroachdb created + clusterrolebinding.rbac.authorization.k8s.io/cockroachdb created + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + +2. As each pod is created, it issues a Certificate Signing Request, or CSR, to have the node's certificate signed by the Kubernetes CA. You must manually check and approve each node's certificates, at which point the CockroachDB node is started in the pod. + + 1. Get the names of the `Pending` CSRs: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get csr + ~~~ + + ~~~ + NAME AGE REQUESTOR CONDITION + default.node.cockroachdb-0 1m system:serviceaccount:default:cockroachdb Pending + default.node.cockroachdb-1 1m system:serviceaccount:default:cockroachdb Pending + default.node.cockroachdb-2 1m system:serviceaccount:default:cockroachdb Pending + ... + ~~~ + + If you do not see a `Pending` CSR, wait a minute and try again. + + 2. Examine the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe csr default.node.cockroachdb-0 + ~~~ + + ~~~ + Name: default.node.cockroachdb-0 + Labels: + Annotations: + CreationTimestamp: Thu, 09 Nov 2017 13:39:37 -0500 + Requesting User: system:serviceaccount:default:cockroachdb + Status: Pending + Subject: + Common Name: node + Serial Number: + Organization: Cockroach + Subject Alternative Names: + DNS Names: localhost + cockroachdb-0.cockroachdb.default.svc.cluster.local + cockroachdb-0.cockroachdb + cockroachdb-public + cockroachdb-public.default.svc.cluster.local + IP Addresses: 127.0.0.1 + 10.48.1.6 + Events: + ~~~ + + 3. If everything looks correct, approve the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.node.cockroachdb-0 + ~~~ + + ~~~ + certificatesigningrequest "default.node.cockroachdb-0" approved + ~~~ + + 4. Repeat steps 2 and 3 for the other 2 pods. + +3. Initialize the CockroachDB cluster: + + 1. Confirm that three pods are `Running` successfully. Note that they will not be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + + 2. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-9e435563-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-0 standard 51m + pvc-9e47d820-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-1 standard 51m + pvc-9e4f57f0-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-2 standard 51m + ~~~ + + 3. Use our [`cluster-init-secure.yaml`](https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init-secure.yaml) file to perform a one-time initialization that joins the CockroachDB nodes into a single cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init-secure.yaml + ~~~ + + ~~~ + job.batch/cluster-init-secure created + ~~~ + + 4. Approve the CSR for the one-off pod from which cluster initialization happens: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.client.root + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.client.root approved + ~~~ + + 5. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get job cluster-init-secure + ~~~ + + ~~~ + NAME COMPLETIONS DURATION AGE + cluster-init-secure 1/1 23s 35s + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cluster-init-secure-q8s7v 0/1 Completed 0 55s + cockroachdb-0 1/1 Running 0 3m + cockroachdb-1 1/1 Running 0 3m + cockroachdb-2 1/1 Running 0 3m + ~~~ + +##### Non-Kubernetes CA + +{{site.data.alerts.callout_info}} +The below steps use [`cockroach cert` commands](cockroach-cert.html) to quickly generate and sign the CockroachDB node and client certificates. Read our [Authentication](authentication.html#using-digital-certificates-with-cockroachdb) docs to learn about other methods of signing certificates. +{{site.data.alerts.end}} + +1. Create two directories: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir my-safe-directory + ~~~ + + Directory | Description + ----------|------------ + `certs` | You'll generate your CA certificate and all node and client certificates and keys in this directory. + `my-safe-directory` | You'll generate your CA key in this directory and then reference the key when generating node and client certificates. + +2. Create the CA certificate and key pair: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +3. Create a client certificate and key pair for the root user: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-client \ + root \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +4. Upload the client certificate and key to the Kubernetes cluster as a secret: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create secret \ + generic cockroachdb.client.root \ + --from-file=certs + ~~~ + + ~~~ + secret/cockroachdb.client.root created + ~~~ + +5. Create the certificate and key pair for your CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + localhost 127.0.0.1 \ + cockroachdb-public \ + cockroachdb-public.default \ + cockroachdb-public.default.svc.cluster.local \ + *.cockroachdb \ + *.cockroachdb.default \ + *.cockroachdb.default.svc.cluster.local \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +6. Upload the node certificate and key to the Kubernetes cluster as a secret: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create secret \ + generic cockroachdb.node \ + --from-file=certs + ~~~ + + ~~~ + secret/cockroachdb.node created + ~~~ + +7. Check that the secrets were created on the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get secrets + ~~~ + + ~~~ + NAME TYPE DATA AGE + cockroachdb.client.root Opaque 3 41m + cockroachdb.node Opaque 5 14s + default-token-6qjdb kubernetes.io/service-account-token 3 4m + ~~~ + +8. Use the config file you downloaded to create the StatefulSet that automatically creates 3 pods, each running a CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset.yaml + ~~~ + + ~~~ + serviceaccount/cockroachdb created + role.rbac.authorization.k8s.io/cockroachdb created + rolebinding.rbac.authorization.k8s.io/cockroachdb created + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + +9. Initialize the CockroachDB cluster: + + 1. Confirm that three pods are `Running` successfully. Note that they will not be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + + 2. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-9e435563-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-0 standard 51m + pvc-9e47d820-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-1 standard 51m + pvc-9e4f57f0-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-2 standard 51m + ~~~ + + 3. Run `cockroach init` on one of the pods to complete the node startup process and have them join together as a cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-0 \ + -- /cockroach/cockroach init \ + --certs-dir=/cockroach/cockroach-certs + ~~~ + + ~~~ + Cluster successfully initialized + ~~~ + + 4. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 1/1 Running 0 3m + cockroachdb-1 1/1 Running 0 3m + cockroachdb-2 1/1 Running 0 3m + ~~~ \ No newline at end of file diff --git a/_includes/v20.2/orchestration/start-cockroachdb-secure.md b/_includes/v20.2/orchestration/start-cockroachdb-secure.md new file mode 100644 index 00000000000..8b46d80cc48 --- /dev/null +++ b/_includes/v20.2/orchestration/start-cockroachdb-secure.md @@ -0,0 +1,401 @@ +Download and modify our StatefulSet configuration, depending on how you want to sign your certificates. + +{{site.data.alerts.callout_danger}} +Some environments, such as Amazon EKS, do not support certificates signed by Kubernetes' built-in CA. In this case, use the second configuration below. +{{site.data.alerts.end}} + +- Using the Kubernetes CA: [`cockroachdb-statefulset-secure.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/cockroachdb-statefulset-secure.yaml). + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cockroachdb-statefulset-secure.yaml + ~~~ + +- Using a non-Kubernetes CA: [`cockroachdb-statefulset.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml) + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/bring-your-own-certs/cockroachdb-statefulset.yaml + ~~~ + +#### Set up configuration file + +Modify the values in the StatefulSet configuration. + +1. To avoid running out of memory when CockroachDB is not the only pod on a Kubernetes node, you *must* set memory limits explicitly. This is because CockroachDB does not detect the amount of memory allocated to its pod when run in Kubernetes. In the `containers` specification, set this amount in both `resources.requests.memory` and `resources.limits.memory`. + + ~~~ + resources: + requests: + memory: "8Gi" + limits: + memory: "8Gi" + ~~~ + + We recommend setting `cache` and `max-sql-memory` each to 1/4 of the memory allocation. These are defined as placeholder percentages in the StatefulSet command that creates the CockroachDB nodes: + + ~~~ + - "exec /cockroach/cockroach start --logtostderr --certs-dir /cockroach/cockroach-certs --advertise-host $(hostname -f) --http-addr 0.0.0.0 --join cockroachdb-0.cockroachdb,cockroachdb-1.cockroachdb,cockroachdb-2.cockroachdb --cache 25% --max-sql-memory 25%" + ~~~ + + {{site.data.alerts.callout_success}} + For example, if you are allocating 8Gi of `memory` to each CockroachDB node, allocate 2Gi to `cache` and 2Gi to `max-sql-memory`. + {{site.data.alerts.end}} + + ~~~ + --cache 2Gi --max-sql-memory 2Gi + ~~~ + +2. In the `volumeClaimTemplates` specification, you may want to modify `resources.requests.storage` for your use case. This configuration defaults to 100Gi of disk space per pod. For more details on customizing disks for performance, see [these instructions](kubernetes-performance.html#disk-type). + + ~~~ + resources: + requests: + storage: "100Gi" + ~~~ + + {{site.data.alerts.callout_info}} + If necessary, you can [expand disk size](orchestrate-cockroachdb-with-kubernetes.html#expand-disk-size) after the cluster is live. + {{site.data.alerts.end}} + +{{site.data.alerts.callout_success}} +If you change the StatefulSet name from the default `cockroachdb`, be sure to start and end with an alphanumeric character and otherwise use lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). +{{site.data.alerts.end}} + +#### Initialize the cluster + +Choose the authentication method that corresponds to the StatefulSet configuration you downloaded and modified above. + +- [Kubernetes CA](#kubernetes-ca) +- [Non-Kubernetes CA](#non-kubernetes-ca) + +{{site.data.alerts.callout_success}} +The StatefulSet configuration sets all CockroachDB nodes to log to `stderr`, so if you ever need access to a pod/node's logs to troubleshoot, use `kubectl logs ` rather than checking the log on the persistent volume. +{{site.data.alerts.end}} + +##### Kubernetes CA + +1. Use the config file you downloaded to create the StatefulSet that automatically creates 3 pods, each running a CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset-secure.yaml + ~~~ + + ~~~ + serviceaccount/cockroachdb created + role.rbac.authorization.k8s.io/cockroachdb created + clusterrole.rbac.authorization.k8s.io/cockroachdb created + rolebinding.rbac.authorization.k8s.io/cockroachdb created + clusterrolebinding.rbac.authorization.k8s.io/cockroachdb created + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + +2. As each pod is created, it issues a Certificate Signing Request, or CSR, to have the node's certificate signed by the Kubernetes CA. You must manually check and approve each node's certificates, at which point the CockroachDB node is started in the pod. + + 1. Get the names of the `Pending` CSRs: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get csr + ~~~ + + ~~~ + NAME AGE REQUESTOR CONDITION + default.node.cockroachdb-0 1m system:serviceaccount:default:cockroachdb Pending + default.node.cockroachdb-1 1m system:serviceaccount:default:cockroachdb Pending + default.node.cockroachdb-2 1m system:serviceaccount:default:cockroachdb Pending + ... + ~~~ + + If you do not see a `Pending` CSR, wait a minute and try again. + + 2. Examine the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl describe csr default.node.cockroachdb-0 + ~~~ + + ~~~ + Name: default.node.cockroachdb-0 + Labels: + Annotations: + CreationTimestamp: Thu, 09 Nov 2017 13:39:37 -0500 + Requesting User: system:serviceaccount:default:cockroachdb + Status: Pending + Subject: + Common Name: node + Serial Number: + Organization: Cockroach + Subject Alternative Names: + DNS Names: localhost + cockroachdb-0.cockroachdb.default.svc.cluster.local + cockroachdb-0.cockroachdb + cockroachdb-public + cockroachdb-public.default.svc.cluster.local + IP Addresses: 127.0.0.1 + 10.48.1.6 + Events: + ~~~ + + 3. If everything looks correct, approve the CSR for the first pod: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.node.cockroachdb-0 + ~~~ + + ~~~ + certificatesigningrequest "default.node.cockroachdb-0" approved + ~~~ + + 4. Repeat steps 2 and 3 for the other 2 pods. + +3. Initialize the CockroachDB cluster: + + 1. Confirm that three pods are `Running` successfully. Note that they will not be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + + 2. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-9e435563-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-0 standard 51m + pvc-9e47d820-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-1 standard 51m + pvc-9e4f57f0-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-2 standard 51m + ~~~ + + 3. Use our [`cluster-init-secure.yaml`](https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init-secure.yaml) file to perform a one-time initialization that joins the CockroachDB nodes into a single cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/cluster-init-secure.yaml + ~~~ + + ~~~ + job.batch/cluster-init-secure created + ~~~ + + 4. Approve the CSR for the one-off pod from which cluster initialization happens: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl certificate approve default.client.root + ~~~ + + ~~~ + certificatesigningrequest.certificates.k8s.io/default.client.root approved + ~~~ + + 5. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get job cluster-init-secure + ~~~ + + ~~~ + NAME COMPLETIONS DURATION AGE + cluster-init-secure 1/1 23s 35s + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cluster-init-secure-q8s7v 0/1 Completed 0 55s + cockroachdb-0 1/1 Running 0 3m + cockroachdb-1 1/1 Running 0 3m + cockroachdb-2 1/1 Running 0 3m + ~~~ + +##### Non-Kubernetes CA + +{{site.data.alerts.callout_info}} +The below steps use [`cockroach cert` commands](cockroach-cert.html) to quickly generate and sign the CockroachDB node and client certificates. Read our [Authentication](authentication.html#using-digital-certificates-with-cockroachdb) docs to learn about other methods of signing certificates. +{{site.data.alerts.end}} + +1. Create two directories: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs my-safe-directory + ~~~ + + Directory | Description + ----------|------------ + `certs` | You'll generate your CA certificate and all node and client certificates and keys in this directory. + `my-safe-directory` | You'll generate your CA key in this directory and then reference the key when generating node and client certificates. + +2. Create the CA certificate and key pair: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +3. Create a client certificate and key pair for the root user: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-client \ + root \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +4. Upload the client certificate and key to the Kubernetes cluster as a secret: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create secret \ + generic cockroachdb.client.root \ + --from-file=certs + ~~~ + + ~~~ + secret/cockroachdb.client.root created + ~~~ + +5. Create the certificate and key pair for your CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + localhost 127.0.0.1 \ + cockroachdb-public \ + cockroachdb-public.default \ + cockroachdb-public.default.svc.cluster.local \ + *.cockroachdb \ + *.cockroachdb.default \ + *.cockroachdb.default.svc.cluster.local \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +6. Upload the node certificate and key to the Kubernetes cluster as a secret: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create secret \ + generic cockroachdb.node \ + --from-file=certs + ~~~ + + ~~~ + secret/cockroachdb.node created + ~~~ + +7. Check that the secrets were created on the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get secrets + ~~~ + + ~~~ + NAME TYPE DATA AGE + cockroachdb.client.root Opaque 3 41m + cockroachdb.node Opaque 5 14s + default-token-6qjdb kubernetes.io/service-account-token 3 4m + ~~~ + +8. Use the config file you downloaded to create the StatefulSet that automatically creates 3 pods, each running a CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f cockroachdb-statefulset.yaml + ~~~ + + ~~~ + serviceaccount/cockroachdb created + role.rbac.authorization.k8s.io/cockroachdb created + rolebinding.rbac.authorization.k8s.io/cockroachdb created + service/cockroachdb-public created + service/cockroachdb created + poddisruptionbudget.policy/cockroachdb-budget created + statefulset.apps/cockroachdb created + ~~~ + +9. Initialize the CockroachDB cluster: + + 1. Confirm that three pods are `Running` successfully. Note that they will not be considered `Ready` until after the cluster has been initialized: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 0/1 Running 0 2m + cockroachdb-1 0/1 Running 0 2m + cockroachdb-2 0/1 Running 0 2m + ~~~ + + 2. Confirm that the persistent volumes and corresponding claims were created successfully for all three pods: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pv + ~~~ + + ~~~ + NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE + pvc-9e435563-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-0 standard 51m + pvc-9e47d820-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-1 standard 51m + pvc-9e4f57f0-fb2e-11e9-a65c-42010a8e0fca 100Gi RWO Delete Bound default/datadir-cockroachdb-2 standard 51m + ~~~ + + 3. Run `cockroach init` on one of the pods to complete the node startup process and have them join together as a cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-0 \ + -- /cockroach/cockroach init \ + --certs-dir=/cockroach/cockroach-certs + ~~~ + + ~~~ + Cluster successfully initialized + ~~~ + + 4. Confirm that cluster initialization has completed successfully. The job should be considered successful and the Kubernetes pods should soon be considered `Ready`: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 1/1 Running 0 3m + cockroachdb-1 1/1 Running 0 3m + cockroachdb-2 1/1 Running 0 3m + ~~~ \ No newline at end of file diff --git a/_includes/v20.2/orchestration/start-kubernetes.md b/_includes/v20.2/orchestration/start-kubernetes.md new file mode 100644 index 00000000000..e8a5a22dc91 --- /dev/null +++ b/_includes/v20.2/orchestration/start-kubernetes.md @@ -0,0 +1,98 @@ +Choose whether you want to orchestrate CockroachDB with Kubernetes using the hosted Google Kubernetes Engine (GKE) service, the hosted Amazon Elastic Kubernetes Service (EKS), or manually on Google Compute Engine (GCE) or AWS. The instructions below will change slightly depending on your choice. + +- [Hosted GKE](#hosted-gke) +- [Hosted EKS](#hosted-eks) +- [Manual GCE](#manual-gce) +- [Manual AWS](#manual-aws) + +### Hosted GKE + +1. Complete the **Before You Begin** steps described in the [Google Kubernetes Engine Quickstart](https://cloud.google.com/kubernetes-engine/docs/quickstart) documentation. + + This includes installing `gcloud`, which is used to create and delete Kubernetes Engine clusters, and `kubectl`, which is the command-line tool used to manage Kubernetes from your workstation. + + {{site.data.alerts.callout_success}}The documentation offers the choice of using Google's Cloud Shell product or using a local shell on your machine. Choose to use a local shell if you want to be able to view the CockroachDB Admin UI using the steps in this guide.{{site.data.alerts.end}} + +2. From your local workstation, start the Kubernetes cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ gcloud container clusters create cockroachdb --machine-type n1-standard-4 + ~~~ + + ~~~ + Creating cluster cockroachdb...done. + ~~~ + + This creates GKE instances and joins them into a single Kubernetes cluster named `cockroachdb`. The `--machine-type` flag tells the node pool to use the [`n1-standard-4`](https://cloud.google.com/compute/docs/machine-types#standard_machine_types) machine type (4 vCPUs, 15 GB memory), which meets our [recommended CPU and memory configuration](recommended-production-settings.html#basic-hardware-recommendations). + + The process can take a few minutes, so do not move on to the next step until you see a `Creating cluster cockroachdb...done` message and details about your cluster. + +3. Get the email address associated with your Google Cloud account: + + {% include copy-clipboard.html %} + ~~~ shell + $ gcloud info | grep Account + ~~~ + + ~~~ + Account: [your.google.cloud.email@example.org] + ~~~ + + {{site.data.alerts.callout_danger}} + This command returns your email address in all lowercase. However, in the next step, you must enter the address using the accurate capitalization. For example, if your address is YourName@example.com, you must use YourName@example.com and not yourname@example.com. + {{site.data.alerts.end}} + +4. [Create the RBAC roles](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control#prerequisites_for_using_role-based_access_control) CockroachDB needs for running on GKE, using the address from the previous step: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create clusterrolebinding $USER-cluster-admin-binding \ + --clusterrole=cluster-admin \ + --user= + ~~~ + + ~~~ + clusterrolebinding.rbac.authorization.k8s.io/your.username-cluster-admin-binding created + ~~~ + +### Hosted EKS + +1. Complete the steps described in the [EKS Getting Started](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html) documentation. + + This includes installing and configuring the AWS CLI and `eksctl`, which is the command-line tool used to create and delete Kubernetes clusters on EKS, and `kubectl`, which is the command-line tool used to manage Kubernetes from your workstation. + +2. From your local workstation, start the Kubernetes cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ eksctl create cluster \ + --name cockroachdb \ + --nodegroup-name standard-workers \ + --node-type m5.xlarge \ + --nodes 3 \ + --nodes-min 1 \ + --nodes-max 4 \ + --node-ami auto + ~~~ + + This creates EKS instances and joins them into a single Kubernetes cluster named `cockroachdb`. The `--node-type` flag tells the node pool to use the [`m5.xlarge`](https://aws.amazon.com/ec2/instance-types/) instance type (4 vCPUs, 16 GB memory), which meets our [recommended CPU and memory configuration](recommended-production-settings.html#basic-hardware-recommendations). + + Cluster provisioning usually takes between 10 and 15 minutes. Do not move on to the next step until you see a message like `[✔] EKS cluster "cockroachdb" in "us-east-1" region is ready` and details about your cluster. + +3. Open the [AWS CloudFormation console](https://console.aws.amazon.com/cloudformation/home) to verify that the stacks `eksctl-cockroachdb-cluster` and `eksctl-cockroachdb-nodegroup-standard-workers` were successfully created. Be sure that your region is selected in the console. + +### Manual GCE + +From your local workstation, install prerequisites and start a Kubernetes cluster as described in the [Running Kubernetes on Google Compute Engine](https://kubernetes.io/docs/setup/turnkey/gce/) documentation. + +The process includes: + +- Creating a Google Cloud Platform account, installing `gcloud`, and other prerequisites. +- Downloading and installing the latest Kubernetes release. +- Creating GCE instances and joining them into a single Kubernetes cluster. +- Installing `kubectl`, the command-line tool used to manage Kubernetes from your workstation. + +### Manual AWS + +From your local workstation, install prerequisites and start a Kubernetes cluster as described in the [Running Kubernetes on AWS EC2](https://kubernetes.io/docs/setup/turnkey/aws/) documentation. diff --git a/_includes/v20.2/orchestration/test-cluster-insecure.md b/_includes/v20.2/orchestration/test-cluster-insecure.md new file mode 100644 index 00000000000..153c8f918f0 --- /dev/null +++ b/_includes/v20.2/orchestration/test-cluster-insecure.md @@ -0,0 +1,72 @@ +1. Launch a temporary interactive pod and start the [built-in SQL client](cockroach-sql.html) inside it: + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=cockroachdb-public + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ kubectl run cockroachdb -it \ + --image=cockroachdb/cockroach:{{page.release_info.version}} \ + --rm \ + --restart=Never \ + -- sql \ + --insecure \ + --host=my-release-cockroachdb-public + ~~~ +
+ +2. Run some basic [CockroachDB SQL statements](learn-cockroachdb-sql.html): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE bank; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bank.accounts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + balance DECIMAL + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bank.accounts (balance) + VALUES + (1000.50), (20000), (380), (500), (55000); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + id | balance + +--------------------------------------+---------+ + 6f123370-c48c-41ff-b384-2c185590af2b | 380 + 990c9148-1ea0-4861-9da7-fd0e65b0a7da | 1000.50 + ac31c671-40bf-4a7b-8bee-452cff8a4026 | 500 + d58afd93-5be9-42ba-b2e2-dc00dcedf409 | 20000 + e6d8f696-87f5-4d3c-a377-8e152fdc27f7 | 55000 + (5 rows) + ~~~ + +3. Exit the SQL shell and delete the temporary pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ diff --git a/_includes/v20.2/orchestration/test-cluster-secure.md b/_includes/v20.2/orchestration/test-cluster-secure.md new file mode 100644 index 00000000000..9cb18c8bf79 --- /dev/null +++ b/_includes/v20.2/orchestration/test-cluster-secure.md @@ -0,0 +1,202 @@ +To use the built-in SQL client, you need to launch a pod that runs indefinitely with the `cockroach` binary inside it, get a shell into the pod, and then start the built-in SQL client. + +
+- Using the Kubernetes CA: [`client-secure.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/client-secure.yaml + ~~~ + +- Using a non-Kubernetes CA: [`client.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/bring-your-own-certs/client.yaml) + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create \ + -f https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/bring-your-own-certs/client.yaml + ~~~ + + {{site.data.alerts.callout_info}} + The pod uses the `root` client certificate created earlier to initialize the cluster, so there's no CSR approval required. If you issue client certificates for other users, however, be sure your SQL usernames contain only lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + + ~~~ + pod/cockroachdb-client-secure created + ~~~ + +1. Get a shell into the pod and start the CockroachDB [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=cockroachdb-public + ~~~ + + ~~~ + # Welcome to the cockroach SQL interface. + # All statements must be terminated by a semicolon. + # To exit: CTRL + D. + # + # Client version: CockroachDB CCL v19.1.0 (x86_64-unknown-linux-gnu, built 2019/04/29 18:36:40, go1.11.6) + # Server version: CockroachDB CCL v19.1.0 (x86_64-unknown-linux-gnu, built 2019/04/29 18:36:40, go1.11.6) + + # Cluster ID: 256a8705-e348-4e3a-ab12-e1aba96857e4 + # + # Enter \? for a brief introduction. + # + root@cockroachdb-public:26257/defaultdb> + ~~~ + +2. Run some basic [CockroachDB SQL statements](learn-cockroachdb-sql.html): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE bank; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bank.accounts VALUES (1, 1000.50); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + id | balance + +----+---------+ + 1 | 1000.50 + (1 row) + ~~~ + +3. [Create a user with a password](create-user.html#create-a-user-with-a-password): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER roach WITH PASSWORD 'Q7gc8rEdS'; + ~~~ + + You will need this username and password to access the Admin UI later. + +4. Exit the SQL shell and pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ +
+ +
+1. From your local workstation, use our [`client-secure.yaml`](https://github.com/cockroachdb/cockroach/blob/master/cloud/kubernetes/client-secure.yaml) file to launch a pod and keep it running indefinitely. + + 1. Download the file: + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -OOOOOOOOO \ + https://raw.githubusercontent.com/cockroachdb/cockroach/master/cloud/kubernetes/client-secure.yaml + ~~~ + + 1. In the file, change `serviceAccountName: cockroachdb` to `serviceAccountName: my-release-cockroachdb`. + + 1. Use the file to launch a pod and keep it running indefinitely: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl create -f client-secure.yaml + ~~~ + + ~~~ + pod "cockroachdb-client-secure" created + ~~~ + + {{site.data.alerts.callout_info}} + The pod uses the `root` client certificate created earlier to initialize the cluster, so there's no CSR approval required. If you issue client certificates for other users, however, be sure your SQL usernames contain only lowercase alphanumeric characters, `-`, or `.` so as to comply with [CSR naming requirements](orchestrate-cockroachdb-with-kubernetes.html#csr-names). + {{site.data.alerts.end}} + +2. Get a shell into the pod and start the CockroachDB [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl exec -it cockroachdb-client-secure \ + -- ./cockroach sql \ + --certs-dir=/cockroach-certs \ + --host=my-release-cockroachdb-public + ~~~ + + ~~~ + # Welcome to the cockroach SQL interface. + # All statements must be terminated by a semicolon. + # To exit: CTRL + D. + # + # Client version: CockroachDB CCL v19.1.0 (x86_64-unknown-linux-gnu, built 2019/04/29 18:36:40, go1.11.6) + # Server version: CockroachDB CCL v19.1.0 (x86_64-unknown-linux-gnu, built 2019/04/29 18:36:40, go1.11.6) + + # Cluster ID: 256a8705-e348-4e3a-ab12-e1aba96857e4 + # + # Enter \? for a brief introduction. + # + root@my-release-cockroachdb-public:26257/defaultdb> + ~~~ + +3. Run some basic [CockroachDB SQL statements](learn-cockroachdb-sql.html): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE bank; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bank.accounts VALUES (1, 1000.50); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + id | balance + +----+---------+ + 1 | 1000.50 + (1 row) + ~~~ + +4. [Create a user with a password](create-user.html#create-a-user-with-a-password): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER roach WITH PASSWORD 'Q7gc8rEdS'; + ~~~ + + You will need this username and password to access the Admin UI later. + +5. Exit the SQL shell and pod: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ +
+ +{{site.data.alerts.callout_success}} +This pod will continue running indefinitely, so any time you need to reopen the built-in SQL client or run any other [`cockroach` client commands](cockroach-commands.html) (e.g., `cockroach node`), repeat step 2 using the appropriate `cockroach` command. + +If you'd prefer to delete the pod and recreate it when needed, run `kubectl delete pod cockroachdb-client-secure`. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/performance/check-rebalancing-after-partitioning.md b/_includes/v20.2/performance/check-rebalancing-after-partitioning.md new file mode 100644 index 00000000000..cbd783fd0b7 --- /dev/null +++ b/_includes/v20.2/performance/check-rebalancing-after-partitioning.md @@ -0,0 +1,41 @@ +Over the next minutes, CockroachDB will rebalance all partitions based on the constraints you defined. + +To check this at a high level, access the Web UI on any node at `:8080` and look at the **Node List**. You'll see that the range count is still close to even across all nodes but much higher than before partitioning: + +Perf tuning rebalancing + +To check at a more granular level, SSH to one of the instances not running CockroachDB and run the `SHOW EXPERIMENTAL_RANGES` statement on the `vehicles` table: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql \ +{{page.certs}} \ +--host=
\ +--database=movr \ +--execute="SELECT * FROM \ +[SHOW EXPERIMENTAL_RANGES FROM TABLE vehicles] \ +WHERE \"start_key\" IS NOT NULL \ + AND \"start_key\" NOT LIKE '%Prefix%';" +~~~ + +~~~ + start_key | end_key | range_id | replicas | lease_holder ++------------------+----------------------------+----------+----------+--------------+ + /"boston" | /"boston"/PrefixEnd | 105 | {1,2,3} | 3 + /"los angeles" | /"los angeles"/PrefixEnd | 121 | {7,8,9} | 8 + /"new york" | /"new york"/PrefixEnd | 101 | {1,2,3} | 3 + /"san francisco" | /"san francisco"/PrefixEnd | 117 | {7,8,9} | 8 + /"seattle" | /"seattle"/PrefixEnd | 113 | {4,5,6} | 5 + /"washington dc" | /"washington dc"/PrefixEnd | 109 | {1,2,3} | 1 +(6 rows) +~~~ + +For reference, here's how the nodes map to zones: + +Node IDs | Zone +---------|----- +1-3 | `us-east1-b` (South Carolina) +4-6 | `us-west1-a` (Oregon) +7-9 | `us-west2-a` (Los Angeles) + +We can see that, after partitioning, the replicas for New York, Boston, and Washington DC are located on nodes 1-3 in `us-east1-b`, replicas for Seattle are located on nodes 4-6 in `us-west1-a`, and replicas for San Francisco and Los Angeles are located on nodes 7-9 in `us-west2-a`. diff --git a/_includes/v20.2/performance/check-rebalancing.md b/_includes/v20.2/performance/check-rebalancing.md new file mode 100644 index 00000000000..fff329ec7cc --- /dev/null +++ b/_includes/v20.2/performance/check-rebalancing.md @@ -0,0 +1,33 @@ +Since you started each node with the `--locality` flag set to its GCE zone, over the next minutes, CockroachDB will rebalance data evenly across the zones. + +To check this, access the Admin UI on any node at `:8080` and look at the **Node List**. You'll see that the range count is more or less even across all nodes: + +Perf tuning rebalancing + +For reference, here's how the nodes map to zones: + +Node IDs | Zone +---------|----- +1-3 | `us-east1-b` (South Carolina) +4-6 | `us-west1-a` (Oregon) +7-9 | `us-west2-a` (Los Angeles) + +To verify even balancing at range level, SSH to one of the instances not running CockroachDB and run the `SHOW EXPERIMENTAL_RANGES` statement: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql \ +{{page.certs}} \ +--host=
\ +--database=movr \ +--execute="SHOW EXPERIMENTAL_RANGES FROM TABLE vehicles;" +~~~ + +~~~ + start_key | end_key | range_id | replicas | lease_holder ++-----------+---------+----------+----------+--------------+ + NULL | NULL | 33 | {3,4,7} | 7 +(1 row) +~~~ + +In this case, we can see that, for the single range containing `vehicles` data, one replica is in each zone, and the leaseholder is in the `us-west2-a` zone. diff --git a/_includes/v20.2/performance/configure-network.md b/_includes/v20.2/performance/configure-network.md new file mode 100644 index 00000000000..7cd3e3cbcc6 --- /dev/null +++ b/_includes/v20.2/performance/configure-network.md @@ -0,0 +1,18 @@ +CockroachDB requires TCP communication on two ports: + +- **26257** (`tcp:26257`) for inter-node communication (i.e., working as a cluster) +- **8080** (`tcp:8080`) for accessing the Admin UI + +Since GCE instances communicate on their internal IP addresses by default, you do not need to take any action to enable inter-node communication. However, to access the Admin UI from your local network, you must [create a firewall rule for your project](https://cloud.google.com/vpc/docs/using-firewalls): + +Field | Recommended Value +------|------------------ +Name | **cockroachweb** +Source filter | IP ranges +Source IP ranges | Your local network's IP ranges +Allowed protocols | **tcp:8080** +Target tags | `cockroachdb` + +{{site.data.alerts.callout_info}} +The **tag** feature will let you easily apply the rule to your instances. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/performance/import-movr.md b/_includes/v20.2/performance/import-movr.md new file mode 100644 index 00000000000..a0fe2dc710a --- /dev/null +++ b/_includes/v20.2/performance/import-movr.md @@ -0,0 +1,160 @@ +Now you'll import Movr data representing users, vehicles, and rides in 3 eastern US cities (New York, Boston, and Washington DC) and 3 western US cities (Los Angeles, San Francisco, and Seattle). + +1. Still on the fourth instance, start the [built-in SQL shell](cockroach-sql.html), pointing it at one of the CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql {{page.certs}} --host=
+ ~~~ + +2. Create the `movr` database and set it as the default: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE movr; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SET DATABASE = movr; + ~~~ + +3. Use the [`IMPORT`](import.html) statement to create and populate the `users`, `vehicles,` and `rides` tables: + + {% include copy-clipboard.html %} + ~~~ sql + > IMPORT TABLE users ( + id UUID NOT NULL, + city STRING NOT NULL, + name STRING NULL, + address STRING NULL, + credit_card STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC) + ) + CSV DATA ( + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/users/n1.0.csv' + ); + ~~~ + + ~~~ + job_id | status | fraction_completed | rows | index_entries | system_records | bytes + +--------------------+-----------+--------------------+------+---------------+----------------+--------+ + 390345990764396545 | succeeded | 1 | 1998 | 0 | 0 | 241052 + (1 row) + + Time: 2.882582355s + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > IMPORT TABLE vehicles ( + id UUID NOT NULL, + city STRING NOT NULL, + type STRING NULL, + owner_id UUID NULL, + creation_time TIMESTAMP NULL, + status STRING NULL, + ext JSON NULL, + mycol STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC) + ) + CSV DATA ( + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/vehicles/n1.0.csv' + ); + ~~~ + + ~~~ + job_id | status | fraction_completed | rows | index_entries | system_records | bytes + +--------------------+-----------+--------------------+-------+---------------+----------------+---------+ + 390346109887250433 | succeeded | 1 | 19998 | 19998 | 0 | 3558767 + (1 row) + + Time: 5.803841493s + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > IMPORT TABLE rides ( + id UUID NOT NULL, + city STRING NOT NULL, + vehicle_city STRING NULL, + rider_id UUID NULL, + vehicle_id UUID NULL, + start_address STRING NULL, + end_address STRING NULL, + start_time TIMESTAMP NULL, + end_time TIMESTAMP NULL, + revenue DECIMAL(10,2) NULL, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + INDEX rides_auto_index_fk_city_ref_users (city ASC, rider_id ASC), + INDEX rides_auto_index_fk_vehicle_city_ref_vehicles (vehicle_city ASC, vehicle_id ASC), + CONSTRAINT check_vehicle_city_city CHECK (vehicle_city = city) + ) + CSV DATA ( + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.0.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.1.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.2.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.3.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.4.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.5.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.6.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.7.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.8.csv', + 'https://s3-us-west-1.amazonaws.com/cockroachdb-movr/datasets/perf-tuning/rides/n1.9.csv' + ); + ~~~ + + ~~~ + job_id | status | fraction_completed | rows | index_entries | system_records | bytes + +--------------------+-----------+--------------------+--------+---------------+----------------+-----------+ + 390346325693792257 | succeeded | 1 | 999996 | 1999992 | 0 | 339741841 + (1 row) + + Time: 44.620371424s + ~~~ + + {{site.data.alerts.callout_success}} + You can observe the progress of imports as well as all schema change operations (e.g., adding secondary indexes) on the [**Jobs** page](admin-ui-jobs-page.html) of the Admin UI. + {{site.data.alerts.end}} + +7. Logically, there should be a number of [foreign key](foreign-key.html) relationships between the tables: + + Referencing columns | Referenced columns + --------------------|------------------- + `vehicles.city`, `vehicles.owner_id` | `users.city`, `users.id` + `rides.city`, `rides.rider_id` | `users.city`, `users.id` + `rides.vehicle_city`, `rides.vehicle_id` | `vehicles.city`, `vehicles.id` + + As mentioned earlier, it wasn't possible to put these relationships in place during `IMPORT`, but it was possible to create the required secondary indexes. Now, let's add the foreign key constraints: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE vehicles + ADD CONSTRAINT fk_city_ref_users + FOREIGN KEY (city, owner_id) + REFERENCES users (city, id); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE rides + ADD CONSTRAINT fk_city_ref_users + FOREIGN KEY (city, rider_id) + REFERENCES users (city, id); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE rides + ADD CONSTRAINT fk_vehicle_city_ref_vehicles + FOREIGN KEY (vehicle_city, vehicle_id) + REFERENCES vehicles (city, id); + ~~~ + +4. Exit the built-in SQL shell: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ diff --git a/_includes/v20.2/performance/overview.md b/_includes/v20.2/performance/overview.md new file mode 100644 index 00000000000..8707c1daf10 --- /dev/null +++ b/_includes/v20.2/performance/overview.md @@ -0,0 +1,38 @@ +### Topology + +You'll start with a 3-node CockroachDB cluster in a single Google Compute Engine (GCE) zone, with an extra instance for running a client application workload: + +Perf tuning topology + +{{site.data.alerts.callout_info}} +Within a single GCE zone, network latency between instances should be sub-millisecond. +{{site.data.alerts.end}} + +You'll then scale the cluster to 9 nodes running across 3 GCE regions, with an extra instance in each region for a client application workload: + +Perf tuning topology + +{{site.data.alerts.callout_info}} +Network latencies will increase with geographic distance between nodes. You can observe this in the [Network Latency page](admin-ui-network-latency-page.html) of the Admin UI. +{{site.data.alerts.end}} + +To reproduce the performance demonstrated in this tutorial: + +- For each CockroachDB node, you'll use the [`n1-standard-4`](https://cloud.google.com/compute/docs/machine-types#standard_machine_types) machine type (4 vCPUs, 15 GB memory) with the Ubuntu 16.04 OS image and a [local SSD](https://cloud.google.com/compute/docs/disks/#localssds) disk. +- For running the client application workload, you'll use smaller instances, such as `n1-standard-1`. + +### Schema + +Your schema and data will be based on our open-source, fictional peer-to-peer vehicle-sharing application, [MovR](movr.html). + +Perf tuning schema + +A few notes about the schema: + +- There are just three self-explanatory tables: In essence, `users` represents the people registered for the service, `vehicles` represents the pool of vehicles for the service, and `rides` represents when and where users have participated. +- Each table has a composite primary key, with `city` being first in the key. Although not necessary initially in the single-region deployment, once you scale the cluster to multiple regions, these compound primary keys will enable you to [geo-partition data at the row level](partitioning.html#partition-using-primary-key) by `city`. As such, this tutorial demonstrates a schema designed for future scaling. +- The [`IMPORT`](import.html) feature you'll use to import the data does not support foreign keys, so you'll import the data without [foreign key constraints](foreign-key.html). However, the import will create the secondary indexes required to add the foreign keys later. + +### Important concepts + +To understand the techniques in this tutorial, and to be able to apply them in your own scenarios, it's important to first understand [how reads and writes work in CockroachDB](architecture/reads-and-writes-overview.html). Review that document before getting started here. diff --git a/_includes/v20.2/performance/partition-by-city.md b/_includes/v20.2/performance/partition-by-city.md new file mode 100644 index 00000000000..d1c4df6e6ec --- /dev/null +++ b/_includes/v20.2/performance/partition-by-city.md @@ -0,0 +1,419 @@ +For this service, the most effective technique for improving read and write latency is to [geo-partition](partitioning.html) the data by city. In essence, this means changing the way data is mapped to ranges. Instead of an entire table and its indexes mapping to a specific range or set of ranges, all rows in the table and its indexes with a given city will map to a range or set of ranges. Once ranges are defined in this way, we can then use the [replication zone](configure-replication-zones.html) feature to pin partitions to specific locations, ensuring that read and write requests from users in a specific city do not have to leave that region. + +1. Partitioning is an enterprise feature, so start off by [registering for a 30-day trial license](https://www.cockroachlabs.com/get-cockroachdb/). + +2. Once you've received the trial license, SSH to any node in your cluster and [apply the license](enterprise-licensing.html#set-a-license): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --host=
\ + --execute="SET CLUSTER SETTING cluster.organization = '';" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --host=
\ + --execute="SET CLUSTER SETTING enterprise.license = '';" + ~~~ + +3. Define partitions for all tables and their secondary indexes. + + Start with the `users` table: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER TABLE users \ + PARTITION BY LIST (city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + Now define partitions for the `vehicles` table and its secondary indexes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER TABLE vehicles \ + PARTITION BY LIST (city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER INDEX vehicles_auto_index_fk_city_ref_users \ + PARTITION BY LIST (city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + Next, define partitions for the `rides` table and its secondary indexes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER TABLE rides \ + PARTITION BY LIST (city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER INDEX rides_auto_index_fk_city_ref_users \ + PARTITION BY LIST (city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="ALTER INDEX rides_auto_index_fk_vehicle_city_ref_vehicles \ + PARTITION BY LIST (vehicle_city) ( \ + PARTITION new_york VALUES IN ('new york'), \ + PARTITION boston VALUES IN ('boston'), \ + PARTITION washington_dc VALUES IN ('washington dc'), \ + PARTITION seattle VALUES IN ('seattle'), \ + PARTITION san_francisco VALUES IN ('san francisco'), \ + PARTITION los_angeles VALUES IN ('los angeles') \ + );" + ~~~ + + Finally, drop an unused index on `rides` rather than partition it: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + {{page.certs}} \ + --database=movr \ + --host=
\ + --execute="DROP INDEX rides_start_time_idx;" + ~~~ + + {{site.data.alerts.callout_info}} + The `rides` table contains 1 million rows, so dropping this index will take a few minutes. + {{site.data.alerts.end}} + +7. Now [create replication zones](configure-replication-zones.html#create-a-replication-zone-for-a-partition) to require city data to be stored on specific nodes based on node locality. + + City | Locality + -----|--------- + New York | `zone=us-east1-b` + Boston | `zone=us-east1-b` + Washington DC | `zone=us-east1-b` + Seattle | `zone=us-west1-a` + San Francisco | `zone=us-west2-a` + Los Angeles | `zone=us-west2-a` + + {{site.data.alerts.callout_info}} + Since our nodes are located in 3 specific GCE zones, we're only going to use the `zone=` portion of node locality. If we were using multiple zones per regions, we would likely use the `region=` portion of the node locality instead. + {{site.data.alerts.end}} + + Start with the `users` table partitions: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF TABLE movr.users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + Move on to the `vehicles` table and secondary index partitions: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF TABLE movr.vehicles CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF INDEX vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + Finish with the `rides` table and secondary index partitions: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION new_york OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION boston OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION washington_dc OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-east1-b]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION seattle OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-west1-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION san_francisco OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF TABLE movr.rides CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF INDEX rides_auto_index_fk_city_ref_users CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER PARTITION los_angeles OF INDEX rides_auto_index_fk_vehicle_city_ref_vehicles CONFIGURE ZONE USING constraints='[+zone=us-west2-a]';" \ + {{page.certs}} \ + --host=
+ ~~~ diff --git a/_includes/v20.2/performance/scale-cluster.md b/_includes/v20.2/performance/scale-cluster.md new file mode 100644 index 00000000000..92aeaddf5b8 --- /dev/null +++ b/_includes/v20.2/performance/scale-cluster.md @@ -0,0 +1,61 @@ +1. SSH to one of the `n1-standard-4` instances in the `us-west1-a` zone. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +3. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + {{page.certs}} \ + --advertise-host= \ + --join= \ + --locality=cloud=gce,region=us-west1,zone=us-west1-a \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +4. Repeat steps 1 - 3 for the other two `n1-standard-4` instances in the `us-west1-a` zone. + +5. SSH to one of the `n1-standard-4` instances in the `us-west2-a` zone. + +6. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +7. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + {{page.certs}} \ + --advertise-host= \ + --join= \ + --locality=cloud=gce,region=us-west2,zone=us-west2-a \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +8. Repeat steps 5 - 7 for the other two `n1-standard-4` instances in the `us-west2-a` zone. diff --git a/_includes/v20.2/performance/start-cluster.md b/_includes/v20.2/performance/start-cluster.md new file mode 100644 index 00000000000..0847b3b268f --- /dev/null +++ b/_includes/v20.2/performance/start-cluster.md @@ -0,0 +1,60 @@ +#### Start the nodes + +1. SSH to the first `n1-standard-4` instance. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +3. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + {{page.certs}} \ + --advertise-host= \ + --join=:26257,:26257,:26257 \ + --locality=cloud=gce,region=us-east1,zone=us-east1-b \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +4. Repeat steps 1 - 3 for the other two `n1-standard-4` instances. Be sure to adjust the `--advertise-addr` flag each time. + +#### Initialize the cluster + +1. SSH to the fourth instance, the one not running a CockroachDB node. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +4. Run the [`cockroach init`](cockroach-init.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init {{page.certs}} --host=
+ ~~~ + + Each node then prints helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the Admin UI, and the SQL URL for clients. diff --git a/_includes/v20.2/performance/test-performance-after-partitioning.md b/_includes/v20.2/performance/test-performance-after-partitioning.md new file mode 100644 index 00000000000..16c07a9f92d --- /dev/null +++ b/_includes/v20.2/performance/test-performance-after-partitioning.md @@ -0,0 +1,93 @@ +After partitioning, reads and writers for a specific city will be much faster because all replicas for that city are now located on the nodes closest to the city. + +To check this, let's repeat a few of the read and write queries that we executed before partitioning in [step 12](#step-12-test-performance). + +#### Reads + +Again imagine we are a Movr administrator in New York, and we want to get the IDs and descriptions of all New York-based bikes that are currently in use: + +1. SSH to the instance in `us-east1-b` with the Python client. + +2. Query for the data: + + {% include copy-clipboard.html %} + ~~~ shell + $ {{page.app}} \ + --host=
\ + --statement="SELECT id, ext FROM vehicles \ + WHERE city = 'new york' \ + AND type = 'bike' \ + AND status = 'in_use'" \ + --repeat=50 \ + --times + ~~~ + + ~~~ + Result: + ['id', 'ext'] + ['0068ee24-2dfb-437d-9a5d-22bb742d519e', "{u'color': u'green', u'brand': u'Kona'}"] + ['01b80764-283b-4232-8961-a8d6a4121a08', "{u'color': u'green', u'brand': u'Pinarello'}"] + ['02a39628-a911-4450-b8c0-237865546f7f', "{u'color': u'black', u'brand': u'Schwinn'}"] + ['02eb2a12-f465-4575-85f8-a4b77be14c54', "{u'color': u'black', u'brand': u'Pinarello'}"] + ['02f2fcc3-fea6-4849-a3a0-dc60480fa6c2', "{u'color': u'red', u'brand': u'FujiCervelo'}"] + ['034d42cf-741f-428c-bbbb-e31820c68588', "{u'color': u'yellow', u'brand': u'Santa Cruz'}"] + ... + + Times (milliseconds): + [20.065784454345703, 7.866144180297852, 8.362054824829102, 9.08803939819336, 7.925987243652344, 7.543087005615234, 7.786035537719727, 8.227825164794922, 7.907867431640625, 7.654905319213867, 7.793903350830078, 7.627964019775391, 7.833957672119141, 7.858037948608398, 7.474184036254883, 9.459972381591797, 7.726192474365234, 7.194995880126953, 7.364034652709961, 7.25102424621582, 7.650852203369141, 7.663965225219727, 9.334087371826172, 7.810115814208984, 7.543087005615234, 7.134914398193359, 7.922887802124023, 7.220029830932617, 7.606029510498047, 7.208108901977539, 7.333993911743164, 7.464170455932617, 7.679939270019531, 7.436990737915039, 7.62486457824707, 7.235050201416016, 7.420063018798828, 7.795095443725586, 7.39598274230957, 7.546901702880859, 7.582187652587891, 7.9669952392578125, 7.418155670166016, 7.539033889770508, 7.805109024047852, 7.086992263793945, 7.069826126098633, 7.833957672119141, 7.43412971496582, 7.035017013549805] + + Median time (milliseconds): + 7.62641429901 + ~~~ + +Before partitioning, this query took a median time of 72.02ms. After partitioning, the query took a median time of only 7.62ms. + +#### Writes + +Now let's again imagine 100 people in New York and 100 people in Seattle and 100 people in New York want to create new Movr accounts: + +1. SSH to the instance in `us-west1-a` with the Python client. + +2. Create 100 Seattle-based users: + + {% include copy-clipboard.html %} + ~~~ shell + {{page.app}} \ + --host=
\ + --statement="INSERT INTO users VALUES (gen_random_uuid(), 'seattle', 'Seatller', '111 East Street', '1736352379937347')" \ + --repeat=100 \ + --times + ~~~ + + ~~~ + Times (milliseconds): + [41.8248176574707, 9.701967239379883, 8.725166320800781, 9.058952331542969, 7.819175720214844, 6.247997283935547, 10.265827178955078, 7.627964019775391, 9.120941162109375, 7.977008819580078, 9.247064590454102, 8.929967880249023, 9.610176086425781, 14.40286636352539, 8.588075637817383, 8.67319107055664, 9.417057037353516, 7.652044296264648, 8.917093276977539, 9.135961532592773, 8.604049682617188, 9.220123291015625, 7.578134536743164, 9.096860885620117, 8.942842483520508, 8.63790512084961, 7.722139358520508, 13.59701156616211, 9.176015853881836, 11.484146118164062, 9.212017059326172, 7.563114166259766, 8.793115615844727, 8.80289077758789, 7.827043533325195, 7.6389312744140625, 17.47584342956543, 9.436845779418945, 7.63392448425293, 8.594989776611328, 9.002208709716797, 8.93402099609375, 8.71896743774414, 8.76307487487793, 8.156061172485352, 8.729934692382812, 8.738040924072266, 8.25190544128418, 8.971929550170898, 7.460832595825195, 8.889198303222656, 8.45789909362793, 8.761167526245117, 10.223865509033203, 8.892059326171875, 8.961915969848633, 8.968114852905273, 7.750988006591797, 7.761955261230469, 9.199142456054688, 9.02700424194336, 9.509086608886719, 9.428977966308594, 7.902860641479492, 8.940935134887695, 8.615970611572266, 8.75401496887207, 7.906913757324219, 8.179187774658203, 11.447906494140625, 8.71419906616211, 9.202003479003906, 9.263038635253906, 9.089946746826172, 8.92496109008789, 10.32114028930664, 7.913827896118164, 9.464025497436523, 10.612010955810547, 8.78596305847168, 8.878946304321289, 7.575035095214844, 10.657072067260742, 8.777856826782227, 8.649110794067383, 9.012937545776367, 8.931875228881836, 9.31406021118164, 9.396076202392578, 8.908987045288086, 8.002996444702148, 9.089946746826172, 7.5588226318359375, 8.918046951293945, 12.117862701416016, 7.266998291015625, 8.074045181274414, 8.955001831054688, 8.868932723999023, 8.755922317504883] + + Median time (milliseconds): + 8.90052318573 + ~~~ + + Before partitioning, this query took a median time of 48.40ms. After partitioning, the query took a median time of only 8.90ms. + +3. SSH to the instance in `us-east1-b` with the Python client. + +4. Create 100 new NY-based users: + + {% include copy-clipboard.html %} + ~~~ shell + {{page.app}} \ + --host=
\ + --statement="INSERT INTO users VALUES (gen_random_uuid(), 'new york', 'New Yorker', '111 West Street', '9822222379937347')" \ + --repeat=100 \ + --times + ~~~ + + ~~~ + Times (milliseconds): + [276.3068675994873, 9.830951690673828, 8.772134780883789, 9.304046630859375, 8.24880599975586, 7.959842681884766, 7.848978042602539, 7.879018783569336, 7.754087448120117, 10.724067687988281, 13.960123062133789, 9.825944900512695, 9.60993766784668, 9.273052215576172, 9.41920280456543, 8.040904998779297, 16.484975814819336, 10.178089141845703, 8.322000503540039, 9.468793869018555, 8.002042770385742, 9.185075759887695, 9.54294204711914, 9.387016296386719, 9.676933288574219, 13.051986694335938, 9.506940841674805, 12.327909469604492, 10.377168655395508, 15.023946762084961, 9.985923767089844, 7.853031158447266, 9.43303108215332, 9.164094924926758, 10.941028594970703, 9.37199592590332, 12.359857559204102, 8.975028991699219, 7.728099822998047, 8.310079574584961, 9.792089462280273, 9.448051452636719, 8.057117462158203, 9.37795639038086, 9.753942489624023, 9.576082229614258, 8.192062377929688, 9.392023086547852, 7.97581672668457, 8.165121078491211, 9.660959243774414, 8.270978927612305, 9.901046752929688, 8.085966110229492, 10.581016540527344, 9.831905364990234, 7.883787155151367, 8.077859878540039, 8.161067962646484, 10.02812385559082, 7.9898834228515625, 9.840965270996094, 9.452104568481445, 9.747028350830078, 9.003162384033203, 9.206056594848633, 9.274005889892578, 7.8449249267578125, 8.827924728393555, 9.322881698608398, 12.08186149597168, 8.76307487487793, 8.353948593139648, 8.182048797607422, 7.736921310424805, 9.31406021118164, 9.263992309570312, 9.282112121582031, 7.823944091796875, 9.11712646484375, 8.099079132080078, 9.156942367553711, 8.363962173461914, 10.974884033203125, 8.729934692382812, 9.2620849609375, 9.27591323852539, 8.272886276245117, 8.25190544128418, 8.093118667602539, 9.259939193725586, 8.413076400756836, 8.198976516723633, 9.95182991027832, 8.024930953979492, 8.895158767700195, 8.243083953857422, 9.076833724975586, 9.994029998779297, 10.149955749511719] + + Median time (milliseconds): + 9.26303863525 + ~~~ + + Before partitioning, this query took a median time of 116.86ms. After partitioning, the query took a median time of only 9.26ms. diff --git a/_includes/v20.2/performance/test-performance.md b/_includes/v20.2/performance/test-performance.md new file mode 100644 index 00000000000..2009ac9653f --- /dev/null +++ b/_includes/v20.2/performance/test-performance.md @@ -0,0 +1,146 @@ +In general, all of the tuning techniques featured in the single-region scenario above still apply in a multi-region deployment. However, the fact that data and leaseholders are spread across the US means greater latencies in many cases. + +#### Reads + +For example, imagine we are a Movr administrator in New York, and we want to get the IDs and descriptions of all New York-based bikes that are currently in use: + +1. SSH to the instance in `us-east1-b` with the Python client. + +2. Query for the data: + + {% include copy-clipboard.html %} + ~~~ shell + $ {{page.app}} \ + --host=
\ + --statement="SELECT id, ext FROM vehicles \ + WHERE city = 'new york' \ + AND type = 'bike' \ + AND status = 'in_use'" \ + --repeat=50 \ + --times + ~~~ + + ~~~ + Result: + ['id', 'ext'] + ['0068ee24-2dfb-437d-9a5d-22bb742d519e', "{u'color': u'green', u'brand': u'Kona'}"] + ['01b80764-283b-4232-8961-a8d6a4121a08', "{u'color': u'green', u'brand': u'Pinarello'}"] + ['02a39628-a911-4450-b8c0-237865546f7f', "{u'color': u'black', u'brand': u'Schwinn'}"] + ['02eb2a12-f465-4575-85f8-a4b77be14c54', "{u'color': u'black', u'brand': u'Pinarello'}"] + ['02f2fcc3-fea6-4849-a3a0-dc60480fa6c2', "{u'color': u'red', u'brand': u'FujiCervelo'}"] + ['034d42cf-741f-428c-bbbb-e31820c68588', "{u'color': u'yellow', u'brand': u'Santa Cruz'}"] + ... + + Times (milliseconds): + [933.8209629058838, 72.02410697937012, 72.45206832885742, 72.39294052124023, 72.8158950805664, 72.07584381103516, 72.21412658691406, 71.96712493896484, 71.75517082214355, 72.16811180114746, 71.78592681884766, 72.91603088378906, 71.91109657287598, 71.4719295501709, 72.40676879882812, 71.8080997467041, 71.84004783630371, 71.98500633239746, 72.40891456604004, 73.75001907348633, 71.45905494689941, 71.53081893920898, 71.46596908569336, 72.07608222961426, 71.94995880126953, 71.41804695129395, 71.29096984863281, 72.11899757385254, 71.63381576538086, 71.3050365447998, 71.83194160461426, 71.20394706726074, 70.9981918334961, 72.79205322265625, 72.63493537902832, 72.15285301208496, 71.8698501586914, 72.30591773986816, 71.53582572937012, 72.69001007080078, 72.03006744384766, 72.56317138671875, 71.61688804626465, 72.17121124267578, 70.20092010498047, 72.12018966674805, 73.34589958190918, 73.01592826843262, 71.49410247802734, 72.19099998474121] + + Median time (milliseconds): + 72.0270872116 + ~~~ + +As we saw earlier, the leaseholder for the `vehicles` table is in `us-west2-a` (Los Angeles), so our query had to go from the gateway node in `us-east1-b` all the way to the west coast and then back again before returning data to the client. + +For contrast, imagine we are now a Movr administrator in Los Angeles, and we want to get the IDs and descriptions of all Los Angeles-based bikes that are currently in use: + +1. SSH to the instance in `us-west2-a` with the Python client. + +2. Query for the data: + + {% include copy-clipboard.html %} + ~~~ shell + $ {{page.app}} \ + --host=
\ + --statement="SELECT id, ext FROM vehicles \ + WHERE city = 'los angeles' \ + AND type = 'bike' \ + AND status = 'in_use'" \ + --repeat=50 \ + --times + ~~~ + + ~~~ + Result: + ['id', 'ext'] + ['00078349-94d4-43e6-92be-8b0d1ac7ee9f', "{u'color': u'blue', u'brand': u'Merida'}"] + ['003f84c4-fa14-47b2-92d4-35a3dddd2d75', "{u'color': u'red', u'brand': u'Kona'}"] + ['0107a133-7762-4392-b1d9-496eb30ee5f9', "{u'color': u'yellow', u'brand': u'Kona'}"] + ['0144498b-4c4f-4036-8465-93a6bea502a3', "{u'color': u'blue', u'brand': u'Pinarello'}"] + ['01476004-fb10-4201-9e56-aadeb427f98a', "{u'color': u'black', u'brand': u'Merida'}"] + + Times (milliseconds): + [782.6759815216064, 8.564949035644531, 8.226156234741211, 7.949113845825195, 7.86590576171875, 7.842063903808594, 7.674932479858398, 7.555961608886719, 7.642984390258789, 8.024930953979492, 7.717132568359375, 8.46409797668457, 7.520914077758789, 7.6541900634765625, 7.458925247192383, 7.671833038330078, 7.740020751953125, 7.771015167236328, 7.598161697387695, 8.411169052124023, 7.408857345581055, 7.469892501831055, 7.524967193603516, 7.764101028442383, 7.750988006591797, 7.2460174560546875, 6.927967071533203, 7.822990417480469, 7.27391242980957, 7.730960845947266, 7.4710845947265625, 7.4310302734375, 7.33494758605957, 7.455110549926758, 7.021188735961914, 7.083892822265625, 7.812976837158203, 7.625102996826172, 7.447957992553711, 7.179021835327148, 7.504940032958984, 7.224082946777344, 7.257938385009766, 7.714986801147461, 7.4939727783203125, 7.6160430908203125, 7.578849792480469, 7.890939712524414, 7.546901702880859, 7.411956787109375] + + Median time (milliseconds): + 7.6071023941 + ~~~ + +Because the leaseholder for `vehicles` is in the same zone as the client request, this query took just 7.60ms compared to the similar query in New York that took 72.02ms. + +#### Writes + +The geographic distribution of data impacts write performance as well. For example, imagine 100 people in Seattle and 100 people in New York want to create new Movr accounts: + +1. SSH to the instance in `us-west1-a` with the Python client. + +2. Create 100 Seattle-based users: + + {% include copy-clipboard.html %} + ~~~ shell + {{page.app}} \ + --host=
\ + --statement="INSERT INTO users VALUES (gen_random_uuid(), 'seattle', 'Seatller', '111 East Street', '1736352379937347')" \ + --repeat=100 \ + --times + ~~~ + + ~~~ + Times (milliseconds): + [277.4538993835449, 50.12702941894531, 47.75214195251465, 48.13408851623535, 47.872066497802734, 48.65407943725586, 47.78695106506348, 49.14689064025879, 52.770137786865234, 49.00097846984863, 48.68602752685547, 47.387123107910156, 47.36208915710449, 47.6841926574707, 46.49209976196289, 47.06096649169922, 46.753883361816406, 46.304941177368164, 48.90894889831543, 48.63715171813965, 48.37393760681152, 49.23295974731445, 50.13418197631836, 48.310041427612305, 48.57516288757324, 47.62911796569824, 47.77693748474121, 47.505855560302734, 47.89996147155762, 49.79205131530762, 50.76479911804199, 50.21500587463379, 48.73299598693848, 47.55592346191406, 47.35088348388672, 46.7071533203125, 43.00808906555176, 43.1060791015625, 46.02813720703125, 47.91092872619629, 68.71294975280762, 49.241065979003906, 48.9039421081543, 47.82295227050781, 48.26998710632324, 47.631025314331055, 64.51892852783203, 48.12812805175781, 67.33417510986328, 48.603057861328125, 50.31013488769531, 51.02396011352539, 51.45716667175293, 50.85396766662598, 49.07512664794922, 47.49894142150879, 44.67201232910156, 43.827056884765625, 44.412851333618164, 46.69189453125, 49.55601692199707, 49.16882514953613, 49.88598823547363, 49.31306838989258, 46.875, 46.69594764709473, 48.31886291503906, 48.378944396972656, 49.0570068359375, 49.417972564697266, 48.22111129760742, 50.662994384765625, 50.58097839355469, 75.44088363647461, 51.05400085449219, 50.85110664367676, 48.187971115112305, 56.7781925201416, 42.47403144836426, 46.2191104888916, 53.96890640258789, 46.697139739990234, 48.99096488952637, 49.1330623626709, 46.34690284729004, 47.09315299987793, 46.39410972595215, 46.51689529418945, 47.58000373840332, 47.924041748046875, 48.426151275634766, 50.22597312927246, 50.1859188079834, 50.37498474121094, 49.861907958984375, 51.477909088134766, 73.09293746948242, 48.779964447021484, 45.13692855834961, 42.2968864440918] + + Median time (milliseconds): + 48.4025478363 + ~~~ + +3. SSH to the instance in `us-east1-b` with the Python client. + +4. Create 100 new NY-based users: + + {% include copy-clipboard.html %} + ~~~ shell + {{page.app}} \ + --host=
\ + --statement="INSERT INTO users VALUES (gen_random_uuid(), 'new york', 'New Yorker', '111 West Street', '9822222379937347')" \ + --repeat=100 \ + --times + ~~~ + + ~~~ + Times (milliseconds): + [131.05082511901855, 116.88899993896484, 115.15498161315918, 117.095947265625, 121.04082107543945, 115.8750057220459, 113.80696296691895, 113.05880546569824, 118.41201782226562, 125.30899047851562, 117.5389289855957, 115.23890495300293, 116.84799194335938, 120.0411319732666, 115.62800407409668, 115.08989334106445, 113.37089538574219, 115.15498161315918, 115.96989631652832, 133.1961154937744, 114.25995826721191, 118.09396743774414, 122.24102020263672, 116.14608764648438, 114.80998992919922, 131.9139003753662, 114.54391479492188, 115.15307426452637, 116.7759895324707, 135.10799407958984, 117.18511581420898, 120.15485763549805, 118.0570125579834, 114.52388763427734, 115.28396606445312, 130.00011444091797, 126.45292282104492, 142.69423484802246, 117.60401725769043, 134.08493995666504, 117.47002601623535, 115.75007438659668, 117.98381805419922, 115.83089828491211, 114.88890647888184, 113.23404312133789, 121.1700439453125, 117.84791946411133, 115.35286903381348, 115.0820255279541, 116.99700355529785, 116.67394638061523, 116.1041259765625, 114.67289924621582, 112.98894882202148, 117.1119213104248, 119.78602409362793, 114.57300186157227, 129.58717346191406, 118.37983131408691, 126.68204307556152, 118.30306053161621, 113.27195167541504, 114.22920227050781, 115.80777168273926, 116.81294441223145, 114.76683616638184, 115.1430606842041, 117.29192733764648, 118.24417114257812, 116.56999588012695, 113.8620376586914, 114.88819122314453, 120.80597877502441, 132.39002227783203, 131.00910186767578, 114.56179618835449, 117.03896522521973, 117.72680282592773, 115.6010627746582, 115.27681350708008, 114.52317237854004, 114.87483978271484, 117.78903007507324, 116.65701866149902, 122.6949691772461, 117.65193939208984, 120.5449104309082, 115.61179161071777, 117.54202842712402, 114.70890045166016, 113.58809471130371, 129.7171115875244, 117.57993698120117, 117.1119213104248, 117.64001846313477, 140.66505432128906, 136.41691207885742, 116.24789237976074, 115.19908905029297] + + Median time (milliseconds): + 116.868495941 + ~~~ + +It took 48.40ms to create a user in Seattle and 116.86ms to create a user in New York. To better understand this discrepancy, let's look at the distribution of data for the `users` table: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql \ +{{page.certs}} \ +--host=
\ +--database=movr \ +--execute="SHOW EXPERIMENTAL_RANGES FROM TABLE users;" +~~~ + +~~~ + start_key | end_key | range_id | replicas | lease_holder ++-----------+---------+----------+----------+--------------+ + NULL | NULL | 49 | {2,6,8} | 6 +(1 row) +~~~ + +For the single range containing `users` data, one replica is in each zone, with the leaseholder in the `us-west1-a` zone. This means that: + +- When creating a user in Seattle, the request doesn't have to leave the zone to reach the leaseholder. However, since a write requires consensus from its replica group, the write has to wait for confirmation from either the replica in `us-west1-b` (Los Angeles) or `us-east1-b` (New York) before committing and then returning confirmation to the client. +- When creating a user in New York, there are more network hops and, thus, increased latency. The request first needs to travel across the continent to the leaseholder in `us-west1-a`. It then has to wait for confirmation from either the replica in `us-west1-b` (Los Angeles) or `us-east1-b` (New York) before committing and then returning confirmation to the client back in the east. diff --git a/_includes/v20.2/performance/tuning-secure.py b/_includes/v20.2/performance/tuning-secure.py new file mode 100644 index 00000000000..a644dbb1c87 --- /dev/null +++ b/_includes/v20.2/performance/tuning-secure.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +import argparse +import psycopg2 +import time + +parser = argparse.ArgumentParser( + description="test performance of statements against movr database") +parser.add_argument("--host", required=True, + help="ip address of one of the CockroachDB nodes") +parser.add_argument("--statement", required=True, + help="statement to execute") +parser.add_argument("--repeat", type=int, + help="number of times to repeat the statement", default = 20) +parser.add_argument("--times", + help="print time for each repetition of the statement", action="store_true") +parser.add_argument("--cumulative", + help="print cumulative time for all repetitions of the statement", action="store_true") +args = parser.parse_args() + +conn = psycopg2.connect( + database='movr', + user='root', + host=args.host, + port=26257, + sslmode='require', + sslrootcert='certs/ca.crt', + sslkey='certs/client.root.key', + sslcert='certs/client.root.crt' +) +conn.set_session(autocommit=True) +cur = conn.cursor() + +def median(lst): + n = len(lst) + if n < 1: + return None + if n % 2 == 1: + return sorted(lst)[n//2] + else: + return sum(sorted(lst)[n//2-1:n//2+1])/2.0 + +times = list() +for n in range(args.repeat): + start = time.time() + statement = args.statement + cur.execute(statement) + if n < 1: + if cur.description is not None: + colnames = [desc[0] for desc in cur.description] + print("") + print("Result:") + print(colnames) + rows = cur.fetchall() + for row in rows: + print([str(cell) for cell in row]) + end = time.time() + times.append((end - start)* 1000) + +cur.close() +conn.close() + +print("") +if args.times: + print("Times (milliseconds):") + print(times) + print("") +# print("Average time (milliseconds):") +# print(float(sum(times))/len(times)) +# print("") +print("Median time (milliseconds):") +print(median(times)) +print("") +if args.cumulative: + print("Cumulative time (milliseconds):") + print(sum(times)) + print("") diff --git a/_includes/v20.2/performance/tuning.py b/_includes/v20.2/performance/tuning.py new file mode 100644 index 00000000000..dcb567dad91 --- /dev/null +++ b/_includes/v20.2/performance/tuning.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import argparse +import psycopg2 +import time + +parser = argparse.ArgumentParser( + description="test performance of statements against movr database") +parser.add_argument("--host", required=True, + help="ip address of one of the CockroachDB nodes") +parser.add_argument("--statement", required=True, + help="statement to execute") +parser.add_argument("--repeat", type=int, + help="number of times to repeat the statement", default = 20) +parser.add_argument("--times", + help="print time for each repetition of the statement", action="store_true") +parser.add_argument("--cumulative", + help="print cumulative time for all repetitions of the statement", action="store_true") +args = parser.parse_args() + +conn = psycopg2.connect( + database='movr', + user='root', + host=args.host, + port=26257 +) +conn.set_session(autocommit=True) +cur = conn.cursor() + +def median(lst): + n = len(lst) + if n < 1: + return None + if n % 2 == 1: + return sorted(lst)[n//2] + else: + return sum(sorted(lst)[n//2-1:n//2+1])/2.0 + +times = list() +for n in range(args.repeat): + start = time.time() + statement = args.statement + cur.execute(statement) + if n < 1: + if cur.description is not None: + colnames = [desc[0] for desc in cur.description] + print("") + print("Result:") + print(colnames) + rows = cur.fetchall() + for row in rows: + print([str(cell) for cell in row]) + end = time.time() + times.append((end - start)* 1000) + +cur.close() +conn.close() + +print("") +if args.times: + print("Times (milliseconds):") + print(times) + print("") +# print("Average time (milliseconds):") +# print(float(sum(times))/len(times)) +# print("") +print("Median time (milliseconds):") +print(median(times)) +print("") +if args.cumulative: + print("Cumulative time (milliseconds):") + print(sum(times)) + print("") diff --git a/_includes/v20.2/performance/use-hash-sharded-indexes.md b/_includes/v20.2/performance/use-hash-sharded-indexes.md new file mode 100644 index 00000000000..ff487520578 --- /dev/null +++ b/_includes/v20.2/performance/use-hash-sharded-indexes.md @@ -0,0 +1 @@ +For performance reasons, we [discourage indexing on sequential keys](indexes.html#indexing-columns). If, however, you are working with a table that must be indexed on sequential keys, you should use [hash-sharded indexes](indexes.html#hash-sharded-indexes). Hash-sharded indexes distribute sequential traffic uniformly across ranges, eliminating single-range hotspots and improving write performance on sequentially-keyed indexes at a small cost to read performance. \ No newline at end of file diff --git a/_includes/v20.2/prod-deployment/advertise-addr-join.md b/_includes/v20.2/prod-deployment/advertise-addr-join.md new file mode 100644 index 00000000000..67019d1fcea --- /dev/null +++ b/_includes/v20.2/prod-deployment/advertise-addr-join.md @@ -0,0 +1,4 @@ +Flag | Description +-----|------------ +`--advertise-addr` | Specifies the IP address/hostname and port to tell other nodes to use. The port number can be omitted, in which case it defaults to `26257`.

This value must route to an IP address the node is listening on (with `--listen-addr` unspecified, the node listens on all IP addresses).

In some networking scenarios, you may need to use `--advertise-addr` and/or `--listen-addr` differently. For more details, see [Networking](recommended-production-settings.html#networking). +`--join` | Identifies the address of 3-5 of the initial nodes of the cluster. These addresses should match the addresses that the target nodes are advertising. diff --git a/_includes/v20.2/prod-deployment/backup.sh b/_includes/v20.2/prod-deployment/backup.sh new file mode 100644 index 00000000000..b1621eeb96a --- /dev/null +++ b/_includes/v20.2/prod-deployment/backup.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +# This script creates full backups when run on the configured +# day of the week and incremental backups when run on other days, and tracks +# recently created backups in a file to pass as the base for incremental backups. + +what="" # Leave empty for full cluster backup, or add "DATABASE database_name" to backup a database. +base="/backups" # The URL where you want to store the backup. +extra="" # Any additional parameters that need to be appended to the BACKUP URI e.g. AWS key params. +recent=recent_backups.txt # File in which recent backups are tracked. +backup_parameters= # e.g. "WITH revision_history" + +# Customize the `cockroach sql` command with `--host`, `--certs-dir` or `--insecure`, `--port`, and additional flags as needed to connect to the SQL client. +runsql() { cockroach sql --insecure -e "$1"; } + +destination="${base}/$(date +"%Y-%V")${extra}" # %V is the week number of the year, with Monday as the first day of the week. + +runsql "BACKUP $what TO '$destination' AS OF SYSTEM TIME '-1m' $backup_parameters" +echo "backed up to ${destination}" diff --git a/_includes/v20.2/prod-deployment/insecure-initialize-cluster.md b/_includes/v20.2/prod-deployment/insecure-initialize-cluster.md new file mode 100644 index 00000000000..b21a1a6fd97 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-initialize-cluster.md @@ -0,0 +1,12 @@ +On your local machine, complete the node startup process and have them join together as a cluster: + +1. [Install CockroachDB](install-cockroachdb.html) on your local machine, if you haven't already. + +2. Run the [`cockroach init`](cockroach-init.html) command, with the `--host` flag set to the address of any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init --insecure --host=
+ ~~~ + + Each node then prints helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the admin UI, and the SQL URL for clients. diff --git a/_includes/v20.2/prod-deployment/insecure-recommendations.md b/_includes/v20.2/prod-deployment/insecure-recommendations.md new file mode 100644 index 00000000000..11bcbe83d83 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-recommendations.md @@ -0,0 +1,13 @@ +- Consider using a [secure cluster](manual-deployment.html) instead. Using an insecure cluster comes with risks: + - Your cluster is open to any client that can access any node's IP addresses. + - Any user, even `root`, can log in without providing a password. + - Any user, connecting as `root`, can read or write any data in your cluster. + - There is no network encryption or authentication, and thus no confidentiality. + +- Decide how you want to access your Admin UI: + + Access Level | Description + -------------|------------ + Partially open | Set a firewall rule to allow only specific IP addresses to communicate on port `8080`. + Completely open | Set a firewall rule to allow all IP addresses to communicate on port `8080`. + Completely closed | Set a firewall rule to disallow all communication on port `8080`. In this case, a machine with SSH access to a node could use an SSH tunnel to access the Admin UI. diff --git a/_includes/v20.2/prod-deployment/insecure-requirements.md b/_includes/v20.2/prod-deployment/insecure-requirements.md new file mode 100644 index 00000000000..170a566be3a --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-requirements.md @@ -0,0 +1,9 @@ +- You must have [SSH access]({{page.ssh-link}}) to each machine. This is necessary for distributing and starting CockroachDB binaries. + +- Your network configuration must allow TCP communication on the following ports: + - `26257` for intra-cluster and client-cluster communication + - `8080` to expose your Admin UI + +- Carefully review the [Production Checklist](recommended-production-settings.html) and recommended [Topology Patterns](topology-patterns.html). + +{% include {{ page.version.version }}/prod-deployment/topology-recommendations.md %} \ No newline at end of file diff --git a/_includes/v20.2/prod-deployment/insecure-scale-cluster.md b/_includes/v20.2/prod-deployment/insecure-scale-cluster.md new file mode 100644 index 00000000000..349f159c2f0 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-scale-cluster.md @@ -0,0 +1,117 @@ +You can start the nodes manually or automate the process using [systemd](https://www.freedesktop.org/wiki/Software/systemd/). + +
+ + +
+

+ +
+ +For each additional node you want to add to the cluster, complete the following steps: + +1. SSH to the machine where you want the node to run. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Run the [`cockroach start`](cockroach-start.html) command, passing the new node's address as the `--advertise-addr` flag and pointing `--join` to the three existing nodes (also include `--locality` if you set it earlier). + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +5. Update your load balancer to recognize the new node. + +
+ +
+ +For each additional node you want to add to the cluster, complete the following steps: + +1. SSH to the machine where you want the node to run. Ensure you are logged in as the `root` user. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Create the Cockroach directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir /var/lib/cockroach + ~~~ + +5. Create a Unix user named `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ useradd cockroach + ~~~ + +6. Change the ownership of `Cockroach` directory to the user `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ chown cockroach /var/lib/cockroach + ~~~ + +7. Download the [sample configuration template](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/insecurecockroachdb.service): + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/insecurecockroachdb.service + ~~~ + + Alternatively, you can create the file yourself and copy the script into it: + + {% include copy-clipboard.html %} + ~~~ shell + {% include {{ page.version.version }}/prod-deployment/insecurecockroachdb.service %} + ~~~ + + Save the file in the `/etc/systemd/system/` directory + +8. Customize the sample configuration template for your deployment: + + Specify values for the following flags in the sample configuration template: + + {% include {{ page.version.version }}/prod-deployment/advertise-addr-join.md %} + +9. Repeat these steps for each additional node that you want in your cluster. + +
diff --git a/_includes/v20.2/prod-deployment/insecure-start-nodes.md b/_includes/v20.2/prod-deployment/insecure-start-nodes.md new file mode 100644 index 00000000000..6f962336a73 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-start-nodes.md @@ -0,0 +1,148 @@ +You can start the nodes manually or automate the process using [systemd](https://www.freedesktop.org/wiki/Software/systemd/). + +
+ + +
+

+ +
+ +For each initial node of your cluster, complete the following steps: + +{{site.data.alerts.callout_info}} +After completing these steps, nodes will not yet be live. They will complete the startup process and join together to form a cluster as soon as the cluster is initialized in the next step. +{{site.data.alerts.end}} + +1. SSH to the machine where you want the node to run. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + + This command primes the node to start, using the following flags: + + Flag | Description + -----|------------ + `--insecure` | Indicates that the cluster is insecure, with no network encryption or authentication. + `--advertise-addr` | Specifies the IP address/hostname and port to tell other nodes to use. The port number can be omitted, in which case it defaults to `26257`.

This value must route to an IP address the node is listening on (with `--listen-addr` unspecified, the node listens on all IP addresses).

In some networking scenarios, you may need to use `--advertise-addr` and/or `--listen-addr` differently. For more details, see [Networking](recommended-production-settings.html#networking). + `--join` | Identifies the address of 3-5 of the initial nodes of the cluster. These addresses should match the addresses that the target nodes are advertising. + `--cache`
`--max-sql-memory` | Increases the node's cache size to 25% of available system memory to improve read performance. The capacity for in-memory SQL processing defaults to 25% of system memory but can be raised, if necessary, to increase the number of simultaneous client connections allowed by the node as well as the node's capacity for in-memory processing of rows when using `ORDER BY`, `GROUP BY`, `DISTINCT`, joins, and window functions. For more details, see [Cache and SQL Memory Size](recommended-production-settings.html#cache-and-sql-memory-size). + `--background` | Starts the node in the background so you gain control of the terminal to issue more commands. + + When deploying across multiple datacenters, or when there is otherwise high latency between nodes, it is recommended to set `--locality` as well. It is also required to use certain enterprise features. For more details, see [Locality](cockroach-start.html#locality). + + For other flags not explicitly set, the command uses default values. For example, the node stores data in `--store=cockroach-data` and binds Admin UI HTTP requests to `--http-addr=localhost:8080`. To set these options manually, see [Start a Node](cockroach-start.html). + +5. Repeat these steps for each additional node that you want in your cluster. + +
+ +
+ +For each initial node of your cluster, complete the following steps: + +{{site.data.alerts.callout_info}}After completing these steps, nodes will not yet be live. They will complete the startup process and join together to form a cluster as soon as the cluster is initialized in the next step.{{site.data.alerts.end}} + +1. SSH to the machine where you want the node to run. Ensure you are logged in as the `root` user. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Create the Cockroach directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir /var/lib/cockroach + ~~~ + +5. Create a Unix user named `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ useradd cockroach + ~~~ + +6. Change the ownership of `Cockroach` directory to the user `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ chown cockroach /var/lib/cockroach + ~~~ + +7. Download the [sample configuration template](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/insecurecockroachdb.service) and save the file in the `/etc/systemd/system/` directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/insecurecockroachdb.service + ~~~ + + Alternatively, you can create the file yourself and copy the script into it: + + {% include copy-clipboard.html %} + ~~~ shell + {% include {{ page.version.version }}/prod-deployment/insecurecockroachdb.service %} + ~~~ + +8. In the sample configuration template, specify values for the following flags: + + {% include {{ page.version.version }}/prod-deployment/advertise-addr-join.md %} + + When deploying across multiple datacenters, or when there is otherwise high latency between nodes, it is recommended to set `--locality` as well. It is also required to use certain enterprise features. For more details, see [Locality](cockroach-start.html#locality). + + For other flags not explicitly set, the command uses default values. For example, the node stores data in `--store=cockroach-data` and binds Admin UI HTTP requests to `--http-port=8080`. To set these options manually, see [Start a Node](cockroach-start.html). + +9. Start the CockroachDB cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ systemctl start insecurecockroachdb + ~~~ + +10. Repeat these steps for each additional node that you want in your cluster. + +{{site.data.alerts.callout_info}} +`systemd` handles node restarts in case of node failure. To stop a node without `systemd` restarting it, run `systemctl stop insecurecockroachdb` +{{site.data.alerts.end}} + +
diff --git a/_includes/v20.2/prod-deployment/insecure-test-cluster.md b/_includes/v20.2/prod-deployment/insecure-test-cluster.md new file mode 100644 index 00000000000..996b3c9a2d7 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-test-cluster.md @@ -0,0 +1,41 @@ +CockroachDB replicates and distributes data behind-the-scenes and uses a [Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol) to enable each node to locate data across the cluster. Once a cluster is live, any node can be used as a SQL gateway. + +When using a load balancer, you should issue commands directly to the load balancer, which then routes traffic to the nodes. + +Use the [built-in SQL client](cockroach-sql.html) locally as follows: + +1. On your local machine, launch the built-in SQL client, with the `--host` flag set to the address of the load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=
+ ~~~ + +2. Create an `insecurenodetest` database: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE insecurenodetest; + ~~~ + +3. View the cluster's databases, which will include `insecurenodetest`: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW DATABASES; + ~~~ + + ~~~ + +--------------------+ + | Database | + +--------------------+ + | crdb_internal | + | information_schema | + | insecurenodetest | + | pg_catalog | + | system | + +--------------------+ + (5 rows) + ~~~ + +4. Use `\q` to exit the SQL shell. diff --git a/_includes/v20.2/prod-deployment/insecure-test-load-balancing.md b/_includes/v20.2/prod-deployment/insecure-test-load-balancing.md new file mode 100644 index 00000000000..9e594e0a864 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecure-test-load-balancing.md @@ -0,0 +1,41 @@ +CockroachDB offers a pre-built `workload` binary for Linux that includes several load generators for simulating client traffic against your cluster. This step features CockroachDB's version of the [TPC-C](http://www.tpc.org/tpcc/) workload. + +{{site.data.alerts.callout_success}}For comprehensive guidance on benchmarking CockroachDB with TPC-C, see our Performance Benchmarking white paper.{{site.data.alerts.end}} + +1. SSH to the machine where you want the run the sample TPC-C workload. + + This should be a machine that is not running a CockroachDB node. + +2. Download `workload` and make it executable: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget https://edge-binaries.cockroachdb.com/cockroach/workload.LATEST ; chmod 755 workload.LATEST + ~~~ + +3. Rename and copy `workload` into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i workload.LATEST /usr/local/bin/workload + ~~~ + +4. Start the TPC-C workload, pointing it at the IP address of the load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ workload run tpcc \ + --drop \ + --init \ + --duration=20m \ + --tolerate-errors \ + "postgresql://root@:26257/tpcc?sslmode=disable" + ~~~ + + This command runs the TPC-C workload against the cluster for 20 minutes, loading 1 "warehouse" of data initially and then issuing about 12 queries per minute via 10 "worker" threads. These workers share SQL connections since individual workers are idle for long periods of time between queries. + + {{site.data.alerts.callout_success}}For more tpcc options, use workload run tpcc --help. For details about other load generators included in workload, use workload run --help. + +4. To monitor the load generator's progress, open the [Admin UI](admin-ui-access-and-navigate.html) by pointing a browser to the address in the `admin` field in the standard output of any node on startup. + + Since the load generator is pointed at the load balancer, the connections will be evenly distributed across nodes. To verify this, click **Metrics** on the left, select the **SQL** dashboard, and then check the **SQL Connections** graph. You can use the **Graph** menu to filter the graph for specific nodes. diff --git a/_includes/v20.2/prod-deployment/insecurecockroachdb.service b/_includes/v20.2/prod-deployment/insecurecockroachdb.service new file mode 100644 index 00000000000..b027b941009 --- /dev/null +++ b/_includes/v20.2/prod-deployment/insecurecockroachdb.service @@ -0,0 +1,16 @@ +[Unit] +Description=Cockroach Database cluster node +Requires=network.target +[Service] +Type=notify +WorkingDirectory=/var/lib/cockroach +ExecStart=/usr/local/bin/cockroach start --insecure --advertise-addr= --join=,, --cache=.25 --max-sql-memory=.25 +TimeoutStopSec=60 +Restart=always +RestartSec=10 +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=cockroach +User=cockroach +[Install] +WantedBy=default.target diff --git a/_includes/v20.2/prod-deployment/monitor-cluster.md b/_includes/v20.2/prod-deployment/monitor-cluster.md new file mode 100644 index 00000000000..cb8185eac19 --- /dev/null +++ b/_includes/v20.2/prod-deployment/monitor-cluster.md @@ -0,0 +1,3 @@ +Despite CockroachDB's various [built-in safeguards against failure](high-availability.html), it is critical to actively monitor the overall health and performance of a cluster running in production and to create alerting rules that promptly send notifications when there are events that require investigation or intervention. + +For details about available monitoring options and the most important events and metrics to alert on, see [Monitoring and Alerting](monitoring-and-alerting.html). diff --git a/_includes/v20.2/prod-deployment/prod-see-also.md b/_includes/v20.2/prod-deployment/prod-see-also.md new file mode 100644 index 00000000000..aa39f71bd9f --- /dev/null +++ b/_includes/v20.2/prod-deployment/prod-see-also.md @@ -0,0 +1,8 @@ +- [Production Checklist](recommended-production-settings.html) +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Monitoring and Alerting](monitoring-and-alerting.html) +- [Performance Benchmarking](performance-benchmarking-with-tpc-c-1k-warehouses.html) +- [Performance Tuning](performance-tuning.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Local Deployment](start-a-local-cluster.html) diff --git a/_includes/v20.2/prod-deployment/secure-generate-certificates.md b/_includes/v20.2/prod-deployment/secure-generate-certificates.md new file mode 100644 index 00000000000..2792ddc6099 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-generate-certificates.md @@ -0,0 +1,201 @@ +You can use either `cockroach cert` commands or [`openssl` commands](create-security-certificates-openssl.html) to generate security certificates. This section features the `cockroach cert` commands. + +Locally, you'll need to [create the following certificates and keys](cockroach-cert.html): + +- A certificate authority (CA) key pair (`ca.crt` and `ca.key`). +- A node key pair for each node, issued to its IP addresses and any common names the machine uses, as well as to the IP addresses and common names for machines running load balancers. +- A client key pair for the `root` user. You'll use this to run a sample workload against the cluster as well as some `cockroach` client commands from your local machine. + +{{site.data.alerts.callout_success}}Before beginning, it's useful to collect each of your machine's internal and external IP addresses, as well as any server names you want to issue certificates for.{{site.data.alerts.end}} + +1. [Install CockroachDB](install-cockroachdb.html) on your local machine, if you haven't already. + +2. Create two directories: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir my-safe-directory + ~~~ + - `certs`: You'll generate your CA certificate and all node and client certificates and keys in this directory and then upload some of the files to your nodes. + - `my-safe-directory`: You'll generate your CA key in this directory and then reference the key when generating node and client certificates. After that, you'll keep the key safe and secret; you will not upload it to your nodes. + +3. Create the CA certificate and key: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +4. Create the certificate and key for the first node, issued to all common names you might use to refer to the node as well as to the load balancer instances: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + \ + \ + \ + \ + localhost \ + 127.0.0.1 \ + \ + \ + \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +5. Upload the CA certificate and node certificate and key to the first node: + + {% if page.title contains "Google" %} + {% include copy-clipboard.html %} + ~~~ shell + $ gcloud compute ssh \ + --project \ + --command "mkdir certs" + ~~~ + + {{site.data.alerts.callout_info}} + `gcloud compute ssh` associates your public SSH key with the GCP project and is only needed when connecting to the first node. See the [GCP docs](https://cloud.google.com/sdk/gcloud/reference/compute/ssh) for more details. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + + {% elsif page.title contains "AWS" %} + {% include copy-clipboard.html %} + ~~~ shell + $ ssh-add /path/.pem + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + + {% else %} + {% include copy-clipboard.html %} + ~~~ shell + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + {% endif %} + +6. Delete the local copy of the node certificate and key: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm certs/node.crt certs/node.key + ~~~ + + {{site.data.alerts.callout_info}} + This is necessary because the certificates and keys for additional nodes will also be named `node.crt` and `node.key`. As an alternative to deleting these files, you can run the next `cockroach cert create-node` commands with the `--overwrite` flag. + {{site.data.alerts.end}} + +7. Create the certificate and key for the second node, issued to all common names you might use to refer to the node as well as to the load balancer instances: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + \ + \ + \ + \ + localhost \ + 127.0.0.1 \ + \ + \ + \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +8. Upload the CA certificate and node certificate and key to the second node: + + {% if page.title contains "AWS" %} + {% include copy-clipboard.html %} + ~~~ shell + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + + {% else %} + {% include copy-clipboard.html %} + ~~~ shell + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + {% endif %} + +9. Repeat steps 6 - 8 for each additional node. + +10. Create a client certificate and key for the `root` user: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-client \ + root \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +11. Upload the CA certificate and client certificate and key to the machine where you will run a sample workload: + + {% include copy-clipboard.html %} + ~~~ shell + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ scp certs/ca.crt \ + certs/client.root.crt \ + certs/client.root.key \ + @:~/certs + ~~~ + + In later steps, you'll also use the `root` user's certificate to run [`cockroach`](cockroach-commands.html) client commands from your local machine. If you might also want to run `cockroach` client commands directly on a node (e.g., for local debugging), you'll need to copy the `root` user's certificate and key to that node as well. + +{{site.data.alerts.callout_info}} +On accessing the Admin UI in a later step, your browser will consider the CockroachDB-created certificate invalid and you’ll need to click through a warning message to get to the UI. You can avoid this issue by [using a certificate issued by a public CA](create-security-certificates-custom-ca.html#accessing-the-admin-ui-for-a-secure-cluster). +{{site.data.alerts.end}} diff --git a/_includes/v20.2/prod-deployment/secure-initialize-cluster.md b/_includes/v20.2/prod-deployment/secure-initialize-cluster.md new file mode 100644 index 00000000000..77443246e9b --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-initialize-cluster.md @@ -0,0 +1,8 @@ +On your local machine, run the [`cockroach init`](cockroach-init.html) command to complete the node startup process and have them join together as a cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --certs-dir=certs --host=
+~~~ + +After running this command, each node prints helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the admin UI, and the SQL URL for clients. diff --git a/_includes/v20.2/prod-deployment/secure-recommendations.md b/_includes/v20.2/prod-deployment/secure-recommendations.md new file mode 100644 index 00000000000..85b0b0b31d0 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-recommendations.md @@ -0,0 +1,7 @@ +- Decide how you want to access your Admin UI: + + Access Level | Description + -------------|------------ + Partially open | Set a firewall rule to allow only specific IP addresses to communicate on port `8080`. + Completely open | Set a firewall rule to allow all IP addresses to communicate on port `8080`. + Completely closed | Set a firewall rule to disallow all communication on port `8080`. In this case, a machine with SSH access to a node could use an SSH tunnel to access the Admin UI. diff --git a/_includes/v20.2/prod-deployment/secure-requirements.md b/_includes/v20.2/prod-deployment/secure-requirements.md new file mode 100644 index 00000000000..78ba1467141 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-requirements.md @@ -0,0 +1,11 @@ +- You must have [CockroachDB installed](install-cockroachdb.html) locally. This is necessary for generating and managing your deployment's certificates. + +- You must have [SSH access]({{page.ssh-link}}) to each machine. This is necessary for distributing and starting CockroachDB binaries. + +- Your network configuration must allow TCP communication on the following ports: + - `26257` for intra-cluster and client-cluster communication + - `8080` to expose your Admin UI + +- Carefully review the [Production Checklist](recommended-production-settings.html) and recommended [Topology Patterns](topology-patterns.html). + +{% include {{ page.version.version }}/prod-deployment/topology-recommendations.md %} \ No newline at end of file diff --git a/_includes/v20.2/prod-deployment/secure-scale-cluster.md b/_includes/v20.2/prod-deployment/secure-scale-cluster.md new file mode 100644 index 00000000000..b86aab81822 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-scale-cluster.md @@ -0,0 +1,124 @@ +You can start the nodes manually or automate the process using [systemd](https://www.freedesktop.org/wiki/Software/systemd/). + +
+ + +
+

+ +
+ +For each additional node you want to add to the cluster, complete the following steps: + +1. SSH to the machine where you want the node to run. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Run the [`cockroach start`](cockroach-start.html) command, passing the new node's address as the `--advertise-addr` flag and pointing `--join` to the three existing nodes (also include `--locality` if you set it earlier). + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +5. Update your load balancer to recognize the new node. + +
+ +
+ +For each additional node you want to add to the cluster, complete the following steps: + +1. SSH to the machine where you want the node to run. Ensure you are logged in as the `root` user. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Create the Cockroach directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir /var/lib/cockroach + ~~~ + +5. Create a Unix user named `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ useradd cockroach + ~~~ + +6. Move the `certs` directory to the `cockroach` directory. + + {% include copy-clipboard.html %} + ~~~ shell + $ mv certs /var/lib/cockroach/ + ~~~ + +7. Change the ownership of `Cockroach` directory to the user `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ chown -R cockroach.cockroach /var/lib/cockroach + ~~~ + +8. Download the [sample configuration template](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/securecockroachdb.service): + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/securecockroachdb.service + ~~~ + + Alternatively, you can create the file yourself and copy the script into it: + + {% include copy-clipboard.html %} + ~~~ shell + {% include {{ page.version.version }}/prod-deployment/securecockroachdb.service %} + ~~~ + + Save the file in the `/etc/systemd/system/` directory. + +9. Customize the sample configuration template for your deployment: + + Specify values for the following flags in the sample configuration template: + + {% include {{ page.version.version }}/prod-deployment/advertise-addr-join.md %} + +10. Repeat these steps for each additional node that you want in your cluster. + +
diff --git a/_includes/v20.2/prod-deployment/secure-start-nodes.md b/_includes/v20.2/prod-deployment/secure-start-nodes.md new file mode 100644 index 00000000000..5a3b441913c --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-start-nodes.md @@ -0,0 +1,153 @@ +You can start the nodes manually or automate the process using [systemd](https://www.freedesktop.org/wiki/Software/systemd/). + +
+ + +
+

+ +
+ +For each initial node of your cluster, complete the following steps: + +{{site.data.alerts.callout_info}}After completing these steps, nodes will not yet be live. They will complete the startup process and join together to form a cluster as soon as the cluster is initialized in the next step.{{site.data.alerts.end}} + +1. SSH to the machine where you want the node to run. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + + This command primes the node to start, using the following flags: + + Flag | Description + -----|------------ + `--certs-dir` | Specifies the directory where you placed the `ca.crt` file and the `node.crt` and `node.key` files for the node. + `--advertise-addr` | Specifies the IP address/hostname and port to tell other nodes to use. The port number can be omitted, in which case it defaults to `26257`.

This value must route to an IP address the node is listening on (with `--listen-addr` unspecified, the node listens on all IP addresses).

In some networking scenarios, you may need to use `--advertise-addr` and/or `--listen-addr` differently. For more details, see [Networking](recommended-production-settings.html#networking). + `--join` | Identifies the address of 3-5 of the initial nodes of the cluster. These addresses should match the addresses that the target nodes are advertising. + `--cache`
`--max-sql-memory` | Increases the node's cache size to 25% of available system memory to improve read performance. The capacity for in-memory SQL processing defaults to 25% of system memory but can be raised, if necessary, to increase the number of simultaneous client connections allowed by the node as well as the node's capacity for in-memory processing of rows when using `ORDER BY`, `GROUP BY`, `DISTINCT`, joins, and window functions. For more details, see [Cache and SQL Memory Size](recommended-production-settings.html#cache-and-sql-memory-size). + `--background` | Starts the node in the background so you gain control of the terminal to issue more commands. + + When deploying across multiple datacenters, or when there is otherwise high latency between nodes, it is recommended to set `--locality` as well. It is also required to use certain enterprise features. For more details, see [Locality](cockroach-start.html#locality). + + For other flags not explicitly set, the command uses default values. For example, the node stores data in `--store=cockroach-data` and binds Admin UI HTTP requests to `--http-addr=:8080`. To set these options manually, see [Start a Node](cockroach-start.html). + +5. Repeat these steps for each additional node that you want in your cluster. + +
+ +
+ +For each initial node of your cluster, complete the following steps: + +{{site.data.alerts.callout_info}}After completing these steps, nodes will not yet be live. They will complete the startup process and join together to form a cluster as soon as the cluster is initialized in the next step.{{site.data.alerts.end}} + +1. SSH to the machine where you want the node to run. Ensure you are logged in as the `root` user. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +3. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +4. Create the Cockroach directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir /var/lib/cockroach + ~~~ + +5. Create a Unix user named `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ useradd cockroach + ~~~ + +6. Move the `certs` directory to the `cockroach` directory. + + {% include copy-clipboard.html %} + ~~~ shell + $ mv certs /var/lib/cockroach/ + ~~~ + +7. Change the ownership of `Cockroach` directory to the user `cockroach`: + + {% include copy-clipboard.html %} + ~~~ shell + $ chown -R cockroach.cockroach /var/lib/cockroach + ~~~ + +8. Download the [sample configuration template](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/securecockroachdb.service) and save the file in the `/etc/systemd/system/` directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/securecockroachdb.service + ~~~ + + Alternatively, you can create the file yourself and copy the script into it: + + {% include copy-clipboard.html %} + ~~~ shell + {% include {{ page.version.version }}/prod-deployment/securecockroachdb.service %} + ~~~ + +9. In the sample configuration template, specify values for the following flags: + + {% include {{ page.version.version }}/prod-deployment/advertise-addr-join.md %} + + When deploying across multiple datacenters, or when there is otherwise high latency between nodes, it is recommended to set `--locality` as well. It is also required to use certain enterprise features. For more details, see [Locality](cockroach-start.html#locality). + + For other flags not explicitly set, the command uses default values. For example, the node stores data in `--store=cockroach-data` and binds Admin UI HTTP requests to `--http-addr=localhost:8080`. To set these options manually, see [Start a Node](cockroach-start.html). + +10. Start the CockroachDB cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ systemctl start securecockroachdb + ~~~ + +11. Repeat these steps for each additional node that you want in your cluster. + +{{site.data.alerts.callout_info}} +`systemd` handles node restarts in case of node failure. To stop a node without `systemd` restarting it, run `systemctl stop securecockroachdb` +{{site.data.alerts.end}} + +
diff --git a/_includes/v20.2/prod-deployment/secure-test-cluster.md b/_includes/v20.2/prod-deployment/secure-test-cluster.md new file mode 100644 index 00000000000..f3a1a4a5e89 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-test-cluster.md @@ -0,0 +1,41 @@ +CockroachDB replicates and distributes data behind-the-scenes and uses a [Gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol) to enable each node to locate data across the cluster. Once a cluster is live, any node can be used as a SQL gateway. + +When using a load balancer, you should issue commands directly to the load balancer, which then routes traffic to the nodes. + +Use the [built-in SQL client](cockroach-sql.html) locally as follows: + +1. On your local machine, launch the built-in SQL client, with the `--host` flag set to the address of the load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs --host=
+ ~~~ + +2. Create a `securenodetest` database: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE securenodetest; + ~~~ + +3. View the cluster's databases, which will include `securenodetest`: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW DATABASES; + ~~~ + + ~~~ + +--------------------+ + | Database | + +--------------------+ + | crdb_internal | + | information_schema | + | securenodetest | + | pg_catalog | + | system | + +--------------------+ + (5 rows) + ~~~ + +4. Use `\q` to exit the SQL shell. \ No newline at end of file diff --git a/_includes/v20.2/prod-deployment/secure-test-load-balancing.md b/_includes/v20.2/prod-deployment/secure-test-load-balancing.md new file mode 100644 index 00000000000..45fb876eaf6 --- /dev/null +++ b/_includes/v20.2/prod-deployment/secure-test-load-balancing.md @@ -0,0 +1,43 @@ +CockroachDB offers a pre-built `workload` binary for Linux that includes several load generators for simulating client traffic against your cluster. This step features CockroachDB's version of the [TPC-C](http://www.tpc.org/tpcc/) workload. + +{{site.data.alerts.callout_success}}For comprehensive guidance on benchmarking CockroachDB with TPC-C, see our Performance Benchmarking white paper.{{site.data.alerts.end}} + +1. SSH to the machine where you want to run the sample TPC-C workload. + + This should be a machine that is not running a CockroachDB node, and it should already have a `certs` directory containing `ca.crt`, `client.root.crt`, and `client.root.key` files. + +2. Download `workload` and make it executable: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget https://edge-binaries.cockroachdb.com/cockroach/workload.LATEST ; chmod 755 workload.LATEST + ~~~ + +3. Rename and copy `workload` into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i workload.LATEST /usr/local/bin/workload + ~~~ + +4. Start the TPC-C workload, pointing it at the IP address of the load balancer and the location of the `ca.crt`, `client.root.crt`, and `client.root.key` files: + + {% include copy-clipboard.html %} + ~~~ shell + $ workload run tpcc \ + --drop \ + --init \ + --duration=20m \ + --tolerate-errors \ + "postgresql://root@:26257/tpcc?sslmode=verify-full&sslrootcert=certs/ca.crt&sslcert=certs/client.root.crt&sslkey=certs/client.root.key" + ~~~ + + This command runs the TPC-C workload against the cluster for 20 minutes, loading 1 "warehouse" of data initially and then issuing about 12 queries per minute via 10 "worker" threads. These workers share SQL connections since individual workers are idle for long periods of time between queries. + + {{site.data.alerts.callout_success}}For more tpcc options, use workload run tpcc --help. For details about other load generators included in workload, use workload run --help. + +5. To monitor the load generator's progress, open the [Admin UI](admin-ui-access-and-navigate.html) by pointing a browser to the address in the `admin` field in the standard output of any node on startup. + + For each user who should have access to the Admin UI for a secure cluster, [create a user with a password](create-user.html#create-a-user-with-a-password) and [assign them to an `admin` role if necessary](admin-ui-overview.html#admin-ui-access). On accessing the Admin UI, the users will see a Login screen, where they will need to enter their usernames and passwords. + + Since the load generator is pointed at the load balancer, the connections will be evenly distributed across nodes. To verify this, click **Metrics** on the left, select the **SQL** dashboard, and then check the **SQL Connections** graph. You can use the **Graph** menu to filter the graph for specific nodes. diff --git a/_includes/v20.2/prod-deployment/securecockroachdb.service b/_includes/v20.2/prod-deployment/securecockroachdb.service new file mode 100644 index 00000000000..39054cf2e1d --- /dev/null +++ b/_includes/v20.2/prod-deployment/securecockroachdb.service @@ -0,0 +1,16 @@ +[Unit] +Description=Cockroach Database cluster node +Requires=network.target +[Service] +Type=notify +WorkingDirectory=/var/lib/cockroach +ExecStart=/usr/local/bin/cockroach start --certs-dir=certs --advertise-addr= --join=,, --cache=.25 --max-sql-memory=.25 +TimeoutStopSec=60 +Restart=always +RestartSec=10 +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=cockroach +User=cockroach +[Install] +WantedBy=default.target diff --git a/_includes/v20.2/prod-deployment/synchronize-clocks.md b/_includes/v20.2/prod-deployment/synchronize-clocks.md new file mode 100644 index 00000000000..9d0fed14d5d --- /dev/null +++ b/_includes/v20.2/prod-deployment/synchronize-clocks.md @@ -0,0 +1,179 @@ +CockroachDB requires moderate levels of [clock synchronization](recommended-production-settings.html#clock-synchronization) to preserve data consistency. For this reason, when a node detects that its clock is out of sync with at least half of the other nodes in the cluster by 80% of the maximum offset allowed (500ms by default), it spontaneously shuts down. This avoids the risk of consistency anomalies, but it's best to prevent clocks from drifting too far in the first place by running clock synchronization software on each node. + +{% if page.title contains "Digital Ocean" or page.title contains "On-Premises" %} + +[`ntpd`](http://doc.ntp.org/) should keep offsets in the single-digit milliseconds, so that software is featured here, but other methods of clock synchronization are suitable as well. + +1. SSH to the first machine. + +2. Disable `timesyncd`, which tends to be active by default on some Linux distributions: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo timedatectl set-ntp no + ~~~ + + Verify that `timesyncd` is off: + + {% include copy-clipboard.html %} + ~~~ shell + $ timedatectl + ~~~ + + Look for `Network time on: no` or `NTP enabled: no` in the output. + +3. Install the `ntp` package: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get install ntp + ~~~ + +4. Stop the NTP daemon: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo service ntp stop + ~~~ + +5. Sync the machine's clock with Google's NTP service: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo ntpd -b time.google.com + ~~~ + + To make this change permanent, in the `/etc/ntp.conf` file, remove or comment out any lines starting with `server` or `pool` and add the following lines: + + {% include copy-clipboard.html %} + ~~~ + server time1.google.com iburst + server time2.google.com iburst + server time3.google.com iburst + server time4.google.com iburst + ~~~ + + Restart the NTP daemon: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo service ntp start + ~~~ + + {{site.data.alerts.callout_info}} + We recommend Google's NTP service because it handles ["smearing" the leap second](https://developers.google.com/time/smear). If you use a different NTP service that doesn't smear the leap second, be sure to configure client-side smearing in the same way on each machine. See the [Production Checklist](recommended-production-settings.html#considerations) for details. + {{site.data.alerts.end}} + +6. Verify that the machine is using a Google NTP server: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo ntpq -p + ~~~ + + The active NTP server will be marked with an asterisk. + +7. Repeat these steps for each machine where a CockroachDB node will run. + +{% elsif page.title contains "Google" %} + +Compute Engine instances are preconfigured to use [NTP](http://www.ntp.org/), which should keep offsets in the single-digit milliseconds. However, Google can’t predict how external NTP services, such as `pool.ntp.org`, will handle the leap second. Therefore, you should: + +- [Configure each GCE instance to use Google's internal NTP service](https://cloud.google.com/compute/docs/instances/managing-instances#configure_ntp_for_your_instances). +- If you plan to run a hybrid cluster across GCE and other cloud providers or environments, note that all of the nodes must be synced to the same time source, or to different sources that implement leap second smearing in the same way. See the [Production Checklist](recommended-production-settings.html#considerations) for details. + +{% elsif page.title contains "AWS" %} + +Amazon provides the [Amazon Time Sync Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html), which uses a fleet of satellite-connected and atomic reference clocks in each AWS Region to deliver accurate current time readings. The service also smears the leap second. + +- [Configure each AWS instance to use the internal Amazon Time Sync Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/set-time.html#configure-amazon-time-service). + - Per the above instructions, ensure that `etc/chrony.conf` on the instance contains the line `server 169.254.169.123 prefer iburst minpoll 4 maxpoll 4` and that other `server` or `pool` lines are commented out. + - To verify that Amazon Time Sync Service is being used, run `chronyc sources -v` and check for a line containing `* 169.254.169.123`. The `*` denotes the preferred time server. +- If you plan to run a hybrid cluster across GCE and other cloud providers or environments, note that all of the nodes must be synced to the same time source, or to different sources that implement leap second smearing in the same way. See the [Production Checklist](recommended-production-settings.html#considerations) for details. + +{% elsif page.title contains "Azure" %} + +[`ntpd`](http://doc.ntp.org/) should keep offsets in the single-digit milliseconds, so that software is featured here. However, to run `ntpd` properly on Azure VMs, it's necessary to first unbind the Time Synchronization device used by the Hyper-V technology running Azure VMs; this device aims to synchronize time between the VM and its host operating system but has been known to cause problems. + +1. SSH to the first machine. + +2. Find the ID of the Hyper-V Time Synchronization device: + + {% include copy-clipboard.html %} + ~~~ shell + $ curl -O https://raw.githubusercontent.com/torvalds/linux/master/tools/hv/lsvmbus + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ python lsvmbus -vv | grep -w "Time Synchronization" -A 3 + ~~~ + + ~~~ + VMBUS ID 12: Class_ID = {9527e630-d0ae-497b-adce-e80ab0175caf} - [Time Synchronization] + Device_ID = {2dd1ce17-079e-403c-b352-a1921ee207ee} + Sysfs path: /sys/bus/vmbus/devices/2dd1ce17-079e-403c-b352-a1921ee207ee + Rel_ID=12, target_cpu=0 + ~~~ + +3. Unbind the device, using the `Device_ID` from the previous command's output: + + {% include copy-clipboard.html %} + ~~~ shell + $ echo | sudo tee /sys/bus/vmbus/drivers/hv_util/unbind + ~~~ + +4. Install the `ntp` package: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get install ntp + ~~~ + +5. Stop the NTP daemon: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo service ntp stop + ~~~ + +6. Sync the machine's clock with Google's NTP service: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo ntpd -b time.google.com + ~~~ + + To make this change permanent, in the `/etc/ntp.conf` file, remove or comment out any lines starting with `server` or `pool` and add the following lines: + + {% include copy-clipboard.html %} + ~~~ + server time1.google.com iburst + server time2.google.com iburst + server time3.google.com iburst + server time4.google.com iburst + ~~~ + + Restart the NTP daemon: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo service ntp start + ~~~ + + {{site.data.alerts.callout_info}} + We recommend Google's NTP service because it handles ["smearing" the leap second](https://developers.google.com/time/smear). If you use a different NTP service that doesn't smear the leap second, be sure to configure client-side smearing in the same way on each machine. See the [Production Checklist](recommended-production-settings.html#considerations) for details. + {{site.data.alerts.end}} + +7. Verify that the machine is using a Google NTP server: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo ntpq -p + ~~~ + + The active NTP server will be marked with an asterisk. + +8. Repeat these steps for each machine where a CockroachDB node will run. + +{% endif %} diff --git a/_includes/v20.2/prod-deployment/topology-recommendations.md b/_includes/v20.2/prod-deployment/topology-recommendations.md new file mode 100644 index 00000000000..05e3cd71a0c --- /dev/null +++ b/_includes/v20.2/prod-deployment/topology-recommendations.md @@ -0,0 +1,13 @@ +- Run each node on a separate machine. Since CockroachDB replicates across nodes, running more than one node per machine increases the risk of data loss if a machine fails. Likewise, if a machine has multiple disks or SSDs, run one node with multiple `--store` flags and not one node per disk. For more details about stores, see [Start a Node](cockroach-start.html#store). + +- When starting each node, use the [`--locality`](cockroach-start.html#locality) flag to describe the node's location, for example, `--locality=region=west,zone=us-west-1`. The key-value pairs should be ordered from most to least inclusive, and the keys and order of key-value pairs must be the same on all nodes. + +- When deploying in a single availability zone: + + - To be able to tolerate the failure of any 1 node, use at least 3 nodes with the [`default` 3-way replication factor](configure-replication-zones.html#view-the-default-replication-zone). In this case, if 1 node fails, each range retains 2 of its 3 replicas, a majority. + + - To be able to tolerate 2 simultaneous node failures, use at least 5 nodes and [increase the `default` replication factor for user data](configure-replication-zones.html#edit-the-default-replication-zone) to 5. The replication factor for [important internal data](configure-replication-zones.html#create-a-replication-zone-for-a-system-range) is 5 by default, so no adjustments are needed for internal data. In this case, if 2 nodes fail at the same time, each range retains 3 of its 5 replicas, a majority. + +- When deploying across multiple availability zones: + - To be able to tolerate the failure of 1 entire AZ in a region, use at least 3 AZs per region and set `--locality` on each node to spread data evenly across regions and AZs. In this case, if 1 AZ goes offline, the 2 remaining AZs retain a majority of replicas. + - To be able to tolerate the failure of 1 entire region, use at least 3 regions. \ No newline at end of file diff --git a/_includes/v20.2/prod-deployment/use-cluster.md b/_includes/v20.2/prod-deployment/use-cluster.md new file mode 100644 index 00000000000..e513a09f046 --- /dev/null +++ b/_includes/v20.2/prod-deployment/use-cluster.md @@ -0,0 +1,11 @@ +Now that your deployment is working, you can: + +1. [Implement your data model](sql-statements.html). +2. [Create users](create-user.html) and [grant them privileges](grant.html). +3. [Connect your application](install-client-drivers.html). Be sure to connect your application to the load balancer, not to a CockroachDB node. + +You may also want to adjust the way the cluster replicates data. For example, by default, a multi-node cluster replicates all data 3 times; you can change this replication factor or create additional rules for replicating individual databases and tables differently. For more information, see [Configure Replication Zones](configure-replication-zones.html). + +{{site.data.alerts.callout_danger}} +When running a cluster of 5 nodes or more, it's safest to [increase the replication factor for important internal data](configure-replication-zones.html#create-a-replication-zone-for-a-system-range) to 5, even if you do not do so for user data. For the cluster as a whole to remain available, the ranges for this internal data must always retain a majority of their replicas. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/aggregates.md b/_includes/v20.2/sql/aggregates.md new file mode 100644 index 00000000000..d74c6ba9c61 --- /dev/null +++ b/_includes/v20.2/sql/aggregates.md @@ -0,0 +1,189 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
array_agg(arg1: bool) → bool[]

Aggregates the selected values into an array.

+
array_agg(arg1: bytes) → bytes[]

Aggregates the selected values into an array.

+
array_agg(arg1: date) → date[]

Aggregates the selected values into an array.

+
array_agg(arg1: decimal) → decimal[]

Aggregates the selected values into an array.

+
array_agg(arg1: float) → float[]

Aggregates the selected values into an array.

+
array_agg(arg1: inet) → inet[]

Aggregates the selected values into an array.

+
array_agg(arg1: int) → int[]

Aggregates the selected values into an array.

+
array_agg(arg1: interval) → interval[]

Aggregates the selected values into an array.

+
array_agg(arg1: string) → string[]

Aggregates the selected values into an array.

+
array_agg(arg1: time) → time[]

Aggregates the selected values into an array.

+
array_agg(arg1: timestamp) → timestamp[]

Aggregates the selected values into an array.

+
array_agg(arg1: timestamptz) → timestamptz[]

Aggregates the selected values into an array.

+
array_agg(arg1: uuid) → uuid[]

Aggregates the selected values into an array.

+
array_agg(arg1: oid) → oid[]

Aggregates the selected values into an array.

+
array_agg(arg1: timetz) → timetz[]

Aggregates the selected values into an array.

+
array_agg(arg1: varbit) → varbit[]

Aggregates the selected values into an array.

+
avg(arg1: decimal) → decimal

Calculates the average of the selected values.

+
avg(arg1: float) → float

Calculates the average of the selected values.

+
avg(arg1: int) → decimal

Calculates the average of the selected values.

+
avg(arg1: interval) → interval

Calculates the average of the selected values.

+
bit_and(arg1: int) → int

Calculates the bitwise AND of all non-null input values, or null if none.

+
bit_and(arg1: varbit) → varbit

Calculates the bitwise AND of all non-null input values, or null if none.

+
bit_or(arg1: int) → int

Calculates the bitwise OR of all non-null input values, or null if none.

+
bit_or(arg1: varbit) → varbit

Calculates the bitwise OR of all non-null input values, or null if none.

+
bool_and(arg1: bool) → bool

Calculates the boolean value of ANDing all selected values.

+
bool_or(arg1: bool) → bool

Calculates the boolean value of ORing all selected values.

+
concat_agg(arg1: bytes) → bytes

Concatenates all selected values.

+
concat_agg(arg1: string) → string

Concatenates all selected values.

+
corr(arg1: float, arg2: float) → float

Calculates the correlation coefficient of the selected values.

+
corr(arg1: float, arg2: int) → float

Calculates the correlation coefficient of the selected values.

+
corr(arg1: int, arg2: float) → float

Calculates the correlation coefficient of the selected values.

+
corr(arg1: int, arg2: int) → float

Calculates the correlation coefficient of the selected values.

+
count(arg1: anyelement) → int

Calculates the number of selected elements.

+
count_rows() → int

Calculates the number of rows.

+
every(arg1: bool) → bool

Calculates the boolean value of ANDing all selected values.

+
json_agg(arg1: anyelement) → jsonb

Aggregates values as a JSON or JSONB array.

+
jsonb_agg(arg1: anyelement) → jsonb

Aggregates values as a JSON or JSONB array.

+
max(arg1: bool) → bool

Identifies the maximum selected value.

+
max(arg1: bytes) → bytes

Identifies the maximum selected value.

+
max(arg1: date) → date

Identifies the maximum selected value.

+
max(arg1: decimal) → decimal

Identifies the maximum selected value.

+
max(arg1: float) → float

Identifies the maximum selected value.

+
max(arg1: inet) → inet

Identifies the maximum selected value.

+
max(arg1: int) → int

Identifies the maximum selected value.

+
max(arg1: interval) → interval

Identifies the maximum selected value.

+
max(arg1: string) → string

Identifies the maximum selected value.

+
max(arg1: time) → time

Identifies the maximum selected value.

+
max(arg1: timestamp) → timestamp

Identifies the maximum selected value.

+
max(arg1: timestamptz) → timestamptz

Identifies the maximum selected value.

+
max(arg1: uuid) → uuid

Identifies the maximum selected value.

+
max(arg1: jsonb) → jsonb

Identifies the maximum selected value.

+
max(arg1: oid) → oid

Identifies the maximum selected value.

+
max(arg1: timetz) → timetz

Identifies the maximum selected value.

+
max(arg1: varbit) → varbit

Identifies the maximum selected value.

+
min(arg1: bool) → bool

Identifies the minimum selected value.

+
min(arg1: bytes) → bytes

Identifies the minimum selected value.

+
min(arg1: date) → date

Identifies the minimum selected value.

+
min(arg1: decimal) → decimal

Identifies the minimum selected value.

+
min(arg1: float) → float

Identifies the minimum selected value.

+
min(arg1: inet) → inet

Identifies the minimum selected value.

+
min(arg1: int) → int

Identifies the minimum selected value.

+
min(arg1: interval) → interval

Identifies the minimum selected value.

+
min(arg1: string) → string

Identifies the minimum selected value.

+
min(arg1: time) → time

Identifies the minimum selected value.

+
min(arg1: timestamp) → timestamp

Identifies the minimum selected value.

+
min(arg1: timestamptz) → timestamptz

Identifies the minimum selected value.

+
min(arg1: uuid) → uuid

Identifies the minimum selected value.

+
min(arg1: jsonb) → jsonb

Identifies the minimum selected value.

+
min(arg1: oid) → oid

Identifies the minimum selected value.

+
min(arg1: timetz) → timetz

Identifies the minimum selected value.

+
min(arg1: varbit) → varbit

Identifies the minimum selected value.

+
sqrdiff(arg1: decimal) → decimal

Calculates the sum of squared differences from the mean of the selected values.

+
sqrdiff(arg1: float) → float

Calculates the sum of squared differences from the mean of the selected values.

+
sqrdiff(arg1: int) → decimal

Calculates the sum of squared differences from the mean of the selected values.

+
stddev(arg1: decimal) → decimal

Calculates the standard deviation of the selected values.

+
stddev(arg1: float) → float

Calculates the standard deviation of the selected values.

+
stddev(arg1: int) → decimal

Calculates the standard deviation of the selected values.

+
stddev_samp(arg1: decimal) → decimal

Calculates the standard deviation of the selected values.

+
stddev_samp(arg1: float) → float

Calculates the standard deviation of the selected values.

+
stddev_samp(arg1: int) → decimal

Calculates the standard deviation of the selected values.

+
string_agg(arg1: bytes, arg2: bytes) → bytes

Concatenates all selected values using the provided delimiter.

+
string_agg(arg1: string, arg2: string) → string

Concatenates all selected values using the provided delimiter.

+
sum(arg1: decimal) → decimal

Calculates the sum of the selected values.

+
sum(arg1: float) → float

Calculates the sum of the selected values.

+
sum(arg1: int) → decimal

Calculates the sum of the selected values.

+
sum(arg1: interval) → interval

Calculates the sum of the selected values.

+
sum_int(arg1: int) → int

Calculates the sum of the selected values.

+
variance(arg1: decimal) → decimal

Calculates the variance of the selected values.

+
variance(arg1: float) → float

Calculates the variance of the selected values.

+
variance(arg1: int) → decimal

Calculates the variance of the selected values.

+
xor_agg(arg1: bytes) → bytes

Calculates the bitwise XOR of the selected values.

+
xor_agg(arg1: int) → int

Calculates the bitwise XOR of the selected values.

+
+ diff --git a/_includes/v20.2/sql/begin-transaction-as-of-system-time-example.md b/_includes/v20.2/sql/begin-transaction-as-of-system-time-example.md new file mode 100644 index 00000000000..ca8735152cd --- /dev/null +++ b/_includes/v20.2/sql/begin-transaction-as-of-system-time-example.md @@ -0,0 +1,19 @@ +{% include copy-clipboard.html %} +~~~ sql +> BEGIN AS OF SYSTEM TIME '2019-04-09 18:02:52.0+00:00'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM products; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> COMMIT; +~~~ diff --git a/_includes/v20.2/sql/combine-alter-table-commands.md b/_includes/v20.2/sql/combine-alter-table-commands.md new file mode 100644 index 00000000000..62839cce017 --- /dev/null +++ b/_includes/v20.2/sql/combine-alter-table-commands.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_success}} +This command can be combined with other `ALTER TABLE` commands in a single statement. For a list of commands that can be combined, see [`ALTER TABLE`](alter-table.html). For a demonstration, see [Add and rename columns atomically](rename-column.html#add-and-rename-columns-atomically). +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/connection-parameters.md b/_includes/v20.2/sql/connection-parameters.md new file mode 100644 index 00000000000..417a19d4e56 --- /dev/null +++ b/_includes/v20.2/sql/connection-parameters.md @@ -0,0 +1,8 @@ +Flag | Description +-----|------------ +`--host` | The server host and port number to connect to. This can be the address of any node in the cluster.

**Env Variable:** `COCKROACH_HOST`
**Default:** `localhost:26257` +`--port`
`-p` | The server port to connect to. Note: The port number can also be specified via `--host`.

**Env Variable:** `COCKROACH_PORT`
**Default:** `26257` +`--user`
`-u` | The [SQL user](create-user.html) that will own the client session.

**Env Variable:** `COCKROACH_USER`
**Default:** `root` +`--insecure` | Use an insecure connection.

**Env Variable:** `COCKROACH_INSECURE`
**Default:** `false` +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html) containing the CA and client certificates and client key.

**Env Variable:** `COCKROACH_CERTS_DIR`
**Default:** `${HOME}/.cockroach-certs/` + `--url` | A [connection URL](connection-parameters.html#connect-using-a-url) to use instead of the other arguments.

**Env Variable:** `COCKROACH_URL`
**Default:** no URL diff --git a/_includes/v20.2/sql/crdb-internal-partitions-example.md b/_includes/v20.2/sql/crdb-internal-partitions-example.md new file mode 100644 index 00000000000..39819038fce --- /dev/null +++ b/_includes/v20.2/sql/crdb-internal-partitions-example.md @@ -0,0 +1,43 @@ +## Querying partitions programmatically + +The `crdb_internal.partitions` internal table contains information about the partitions in your database. In testing, scripting, and other programmatic environments, we recommend querying this table for partition information instead of using the `SHOW PARTITIONS` statement. For example, to get all `us_west` partitions of in your database, you can run the following query: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM crdb_internal.partitions WHERE name='us_west'; +~~~ + +~~~ + table_id | index_id | parent_name | name | columns | column_names | list_value | range_value | zone_id | subzone_id ++----------+----------+-------------+---------+---------+--------------+-------------------------------------------------+-------------+---------+------------+ + 53 | 1 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 0 | 0 + 54 | 1 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 54 | 1 + 54 | 2 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 54 | 2 + 55 | 1 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 55 | 1 + 55 | 2 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 55 | 2 + 55 | 3 | NULL | us_west | 1 | vehicle_city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 55 | 3 + 56 | 1 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 56 | 1 + 58 | 1 | NULL | us_west | 1 | city | ('seattle'), ('san francisco'), ('los angeles') | NULL | 58 | 1 +(8 rows) +~~~ + +Other internal tables, like `crdb_internal.tables`, include information that could be useful in conjunction with `crdb_internal.partitions`. + +For example, if you want the output for your partitions to include the name of the table and database, you can perform a join of the two tables: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT + partitions.name AS partition_name, column_names, list_value, tables.name AS table_name, database_name + FROM crdb_internal.partitions JOIN crdb_internal.tables ON partitions.table_id=tables.table_id + WHERE tables.name='users'; +~~~ + +~~~ + partition_name | column_names | list_value | table_name | database_name ++----------------+--------------+-------------------------------------------------+------------+---------------+ + us_west | city | ('seattle'), ('san francisco'), ('los angeles') | users | movr + us_east | city | ('new york'), ('boston'), ('washington dc') | users | movr + europe_west | city | ('amsterdam'), ('paris'), ('rome') | users | movr +(3 rows) +~~~ diff --git a/_includes/v20.2/sql/crdb-internal-partitions.md b/_includes/v20.2/sql/crdb-internal-partitions.md new file mode 100644 index 00000000000..ebab5abe4ed --- /dev/null +++ b/_includes/v20.2/sql/crdb-internal-partitions.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_success}} +In testing, scripting, and other programmatic environments, we recommend querying the `crdb_internal.partitions` internal table for partition information instead of using the `SHOW PARTITIONS` statement. For more information, see [Querying partitions programmatically](show-partitions.html#querying-partitions-programmatically). +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/diagrams/add_column.html b/_includes/v20.2/sql/diagrams/add_column.html new file mode 100644 index 00000000000..f59fd135d0e --- /dev/null +++ b/_includes/v20.2/sql/diagrams/add_column.html @@ -0,0 +1,52 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +ADD + + +COLUMN + + +IF + + +NOT + + +EXISTS + + + +column_name + + + + +typename + + + + +col_qualification + + + + +
diff --git a/_includes/v20.2/sql/diagrams/add_constraint.html b/_includes/v20.2/sql/diagrams/add_constraint.html new file mode 100644 index 00000000000..a8f3b1c9c61 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/add_constraint.html @@ -0,0 +1,38 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +ADD + + +CONSTRAINT + + + +constraint_name + + + + +constraint_elem + + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_column.html b/_includes/v20.2/sql/diagrams/alter_column.html new file mode 100644 index 00000000000..538c7895cd9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_column.html @@ -0,0 +1,90 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + +ALTER + + +COLUMN + + +column_name + +SET + + +DEFAULT + + +a_expr + +NOT + + +NULL + + +DATA + + +TYPE + + +typename + +COLLATE + + +collation_name + +USING + + +a_expr + +DROP + + +DEFAULT + + +NOT + + +NULL + + +STORED + + +TYPE + + +typename + +COLLATE + + +collation_name + +USING + + +a_expr + +
diff --git a/_includes/v20.2/sql/diagrams/alter_index_partition_by.html b/_includes/v20.2/sql/diagrams/alter_index_partition_by.html new file mode 100644 index 00000000000..55136f4ad23 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_index_partition_by.html @@ -0,0 +1,72 @@ +
+ + + + +ALTER + + +INDEX + + +IF + + +EXISTS + + +table_name + +@ + + +index_name + +PARTITION + + +BY + + +LIST + + +( + + +name_list + +) + + +( + + +list_partitions + +RANGE + + +( + + +name_list + +) + + +( + + +range_partitions + +) + + +NOTHING + + +, + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_primary_key.html b/_includes/v20.2/sql/diagrams/alter_primary_key.html new file mode 100644 index 00000000000..5996d6b8681 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_primary_key.html @@ -0,0 +1,71 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +ALTER + + +PRIMARY + + +KEY + + +USING + + +COLUMNS + + +( + + + +index_params + + + +) + + +USING + + +HASH + + +WITH + + +BUCKET_COUNT + + += + + +n_buckets + + + +opt_interleave + + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_role.html b/_includes/v20.2/sql/diagrams/alter_role.html new file mode 100644 index 00000000000..6291ec7cca0 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_role.html @@ -0,0 +1,29 @@ +
+ + + + +ALTER + + +ROLE + + +USER + + +IF + + +EXISTS + + +name + + +opt_with + + +role_options + +
diff --git a/_includes/v20.2/sql/diagrams/alter_sequence_options.html b/_includes/v20.2/sql/diagrams/alter_sequence_options.html new file mode 100644 index 00000000000..7d3068e25e9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_sequence_options.html @@ -0,0 +1,72 @@ +
+ + + + +ALTER + + +SEQUENCE + + +IF + + +EXISTS + + +sequence_name + + +NO + + +CYCLE + + +MINVALUE + + +MAXVALUE + + +OWNED + + +BY + + +NONE + + + +column_name + + + +INCREMENT + + +BY + + +MINVALUE + + +MAXVALUE + + +START + + +WITH + + +integer + + +VIRTUAL + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_table.html b/_includes/v20.2/sql/diagrams/alter_table.html new file mode 100644 index 00000000000..47092b1ed90 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_table.html @@ -0,0 +1,255 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +RENAME + + +COLUMN + + +CONSTRAINT + + + +column_name + + + +TO + + + +column_name + + + +ADD + + +COLUMN + + +IF + + +NOT + + +EXISTS + + + +column_name + + + + +typename + + + + +col_qual_list + + + +CONSTRAINT + + + +constraint_name + + + + +constraint_elem + + + +ALTER + + +COLUMN + + + +column_name + + + +SET + + +DEFAULT + + + +a_expr + + + +NOT + + +NULL + + +DROP + + +DEFAULT + + +NOT + + +NULL + + +STORED + + +SET + + +DATA + + +TYPE + + + +typename + + + +COLLATE + + + +collation_name + + + +USING + + + +a_expr + + + +PRIMARY + + +KEY + + +USING + + +COLUMNS + + +( + + + +index_params + + + +) + + + +opt_interleave + + + +DROP + + +COLUMN + + +IF + + +EXISTS + + + +column_name + + + +CONSTRAINT + + +IF + + +EXISTS + + + +constraint_name + + + +CASCADE + + +RESTRICT + + +VALIDATE + + +CONSTRAINT + + + +constraint_name + + + +EXPERIMENTAL_AUDIT + + +SET + + + +audit_mode + + + + +partition_by + + + +, + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_table_partition_by.html b/_includes/v20.2/sql/diagrams/alter_table_partition_by.html new file mode 100644 index 00000000000..073c8794394 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_table_partition_by.html @@ -0,0 +1,81 @@ +
+ + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + table_name + + + + PARTITION + + + BY + + + LIST + + + ( + + + + name_list + + + + ) + + + ( + + + + list_partitions + + + + RANGE + + + ( + + + + name_list + + + + ) + + + ( + + + + range_partitions + + + + ) + + + NOTHING + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/alter_type.html b/_includes/v20.2/sql/diagrams/alter_type.html new file mode 100644 index 00000000000..ace962f3b99 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_type.html @@ -0,0 +1,45 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +ALTER + + +COLUMN + + +column_name + + +SET + + +DATA + + +TYPE + + + +typename + + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_user_password.html b/_includes/v20.2/sql/diagrams/alter_user_password.html new file mode 100644 index 00000000000..0e014933d1b --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_user_password.html @@ -0,0 +1,31 @@ +
+ + + + +ALTER + + +USER + + +IF + + +EXISTS + + +name + + +WITH + + +PASSWORD + + +password + + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_view.html b/_includes/v20.2/sql/diagrams/alter_view.html new file mode 100644 index 00000000000..2e481fa60aa --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_view.html @@ -0,0 +1,36 @@ +
+ + + + + + ALTER + + + VIEW + + + IF + + + EXISTS + + + + view_name + + + + RENAME + + + TO + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/alter_zone_database.html b/_includes/v20.2/sql/diagrams/alter_zone_database.html new file mode 100644 index 00000000000..11eeb471abb --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_zone_database.html @@ -0,0 +1,61 @@ +
+ + + + +ALTER + + +DATABASE + + +database_name + +CONFIGURE + + +ZONE + + +USING + + +variable + += + + +COPY + + +FROM + + +PARENT + + +value + +, + + +variable + += + + +value + +COPY + + +FROM + + +PARENT + + +DISCARD + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_zone_index.html b/_includes/v20.2/sql/diagrams/alter_zone_index.html new file mode 100644 index 00000000000..ef64e2314d3 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_zone_index.html @@ -0,0 +1,66 @@ +
+ + + + +ALTER + + +INDEX + + +table_name + +@ + + +index_name + +CONFIGURE + + +ZONE + + +USING + + +variable + += + + +COPY + + +FROM + + +PARENT + + +value + +, + + +variable + += + + +value + +COPY + + +FROM + + +PARENT + + +DISCARD + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_zone_partition.html b/_includes/v20.2/sql/diagrams/alter_zone_partition.html new file mode 100644 index 00000000000..69ee2d0eb57 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_zone_partition.html @@ -0,0 +1,84 @@ +
+ + + + +ALTER + + +PARTITION + + +partition_name + +OF + + +TABLE + + +table_name + +INDEX + + +table_name + +@ + + +index_name + +* + + +index_name + +CONFIGURE + + +ZONE + + +USING + + +variable + += + + +COPY + + +FROM + + +PARENT + + +value + +, + + +variable + += + + +value + +COPY + + +FROM + + +PARENT + + +DISCARD + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_zone_range.html b/_includes/v20.2/sql/diagrams/alter_zone_range.html new file mode 100644 index 00000000000..890dcc7240c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_zone_range.html @@ -0,0 +1,61 @@ +
+ + + + +ALTER + + +RANGE + + +range_name + +CONFIGURE + + +ZONE + + +USING + + +variable + += + + +COPY + + +FROM + + +PARENT + + +value + +, + + +variable + += + + +value + +COPY + + +FROM + + +PARENT + + +DISCARD + + +
diff --git a/_includes/v20.2/sql/diagrams/alter_zone_table.html b/_includes/v20.2/sql/diagrams/alter_zone_table.html new file mode 100644 index 00000000000..11c233ebc84 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/alter_zone_table.html @@ -0,0 +1,61 @@ +
+ + + + +ALTER + + +TABLE + + +table_name + +CONFIGURE + + +ZONE + + +USING + + +variable + += + + +COPY + + +FROM + + +PARENT + + +value + +, + + +variable + += + + +value + +COPY + + +FROM + + +PARENT + + +DISCARD + + +
diff --git a/_includes/v20.2/sql/diagrams/backup.html b/_includes/v20.2/sql/diagrams/backup.html new file mode 100644 index 00000000000..b2e6b998113 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/backup.html @@ -0,0 +1,50 @@ +
+ + + + +BACKUP + + +TABLE + + +table_pattern + +, + + +DATABASE + + +database_name + +, + + +TO + + +destination + +AS OF SYSTEM TIME + + +timestamp + +INCREMENTAL FROM + + +full_backup_location + +, + + +incremental_backup_location + +WITH + + +kv_option_list + +
diff --git a/_includes/v20.2/sql/diagrams/begin_transaction.html b/_includes/v20.2/sql/diagrams/begin_transaction.html new file mode 100644 index 00000000000..7e40de65c56 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/begin_transaction.html @@ -0,0 +1,50 @@ +
+ + + + +BEGIN + + +TRANSACTION + + +PRIORITY + + +LOW + + +NORMAL + + +HIGH + + +READ + + +ONLY + + +WRITE + + +AS + + +OF + + +SYSTEM + + +TIME + + +a_expr + +, + + +
diff --git a/_includes/v20.2/sql/diagrams/cancel.html b/_includes/v20.2/sql/diagrams/cancel.html new file mode 100644 index 00000000000..5091140ae13 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/cancel.html @@ -0,0 +1,19 @@ +
+ + + + + + CANCEL + + + QUERY + + + + a_expr + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/cancel_job.html b/_includes/v20.2/sql/diagrams/cancel_job.html new file mode 100644 index 00000000000..e8cbeb150fe --- /dev/null +++ b/_includes/v20.2/sql/diagrams/cancel_job.html @@ -0,0 +1,24 @@ +
+ + + + +CANCEL + + +JOB + + +job_id + + +JOBS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/cancel_query.html b/_includes/v20.2/sql/diagrams/cancel_query.html new file mode 100644 index 00000000000..612db072eb4 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/cancel_query.html @@ -0,0 +1,36 @@ +
+ + + + +CANCEL + + +QUERY + + +IF + + +EXISTS + + +query_id + + +QUERIES + + +IF + + +EXISTS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/cancel_session.html b/_includes/v20.2/sql/diagrams/cancel_session.html new file mode 100644 index 00000000000..857f87adb18 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/cancel_session.html @@ -0,0 +1,36 @@ +
+ + + + +CANCEL + + +SESSION + + +IF + + +EXISTS + + +session_id + + +SESSIONS + + +IF + + +EXISTS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/check_column_level.html b/_includes/v20.2/sql/diagrams/check_column_level.html new file mode 100644 index 00000000000..59eec3e3c15 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/check_column_level.html @@ -0,0 +1,70 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + CHECK + + + ( + + + + check_expr + + + + ) + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/check_table_level.html b/_includes/v20.2/sql/diagrams/check_table_level.html new file mode 100644 index 00000000000..6066d637220 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/check_table_level.html @@ -0,0 +1,60 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_def + + + + , + + + CONSTRAINT + + + + name + + + + CHECK + + + ( + + + + check_expr + + + + ) + + + + table_constraints + + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/col_qual_list.html b/_includes/v20.2/sql/diagrams/col_qual_list.html new file mode 100644 index 00000000000..290034152a6 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/col_qual_list.html @@ -0,0 +1,115 @@ +
+ + + + + + CONSTRAINT + + + + name + + + + NOT + + + NULL + + + UNIQUE + + + PRIMARY + + + KEY + + + CHECK + + + ( + + + + a_expr + + + + ) + + + DEFAULT + + + + b_expr + + + + REFERENCES + + + + qualified_name + + + + + opt_name_parens + + + + + key_actions + + + + COLLATE + + + + unrestricted_name + + + + FAMILY + + + + name + + + + CREATE + + + FAMILY + + + + opt_name + + + + IF + + + NOT + + + EXISTS + + + FAMILY + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/col_qualification.html b/_includes/v20.2/sql/diagrams/col_qualification.html new file mode 100644 index 00000000000..71573b90314 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/col_qualification.html @@ -0,0 +1,154 @@ +
+ + + + +CONSTRAINT + + + +constraint_name + + + +NOT + + +NULL + + +UNIQUE + + +PRIMARY + + +KEY + + +USING + + +HASH + + +WITH + + +BUCKET_COUNT + + += + + +n_buckets + + +CHECK + + +( + + + +a_expr + + + +) + + +DEFAULT + + + +b_expr + + + +REFERENCES + + + +table_name + + + + +opt_name_parens + + + + +key_match + + + + +reference_actions + + + +AS + + +( + + + +a_expr + + + +) + + +STORED + + +COLLATE + + + +collation_name + + + +FAMILY + + + +family_name + + + +CREATE + + +FAMILY + + + +family_name + + + +IF + + +NOT + + +EXISTS + + +FAMILY + + + +family_name + + + + +
diff --git a/_includes/v20.2/sql/diagrams/column_def.html b/_includes/v20.2/sql/diagrams/column_def.html new file mode 100644 index 00000000000..284e8dc5838 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/column_def.html @@ -0,0 +1,23 @@ + \ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/comment.html b/_includes/v20.2/sql/diagrams/comment.html new file mode 100644 index 00000000000..d9933514585 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/comment.html @@ -0,0 +1,53 @@ +
+ + + + +COMMENT + + +ON + + +DATABASE + + + +database_name + + + +TABLE + + + +table_name + + + +COLUMN + + + +column_name + + + +INDEX + + + +table_index_name + + + +IS + + + +comment_text + + + + +
diff --git a/_includes/v20.2/sql/diagrams/commit_transaction.html b/_includes/v20.2/sql/diagrams/commit_transaction.html new file mode 100644 index 00000000000..12914f3e1cb --- /dev/null +++ b/_includes/v20.2/sql/diagrams/commit_transaction.html @@ -0,0 +1,17 @@ +
+ + + + + + COMMIT + + + END + + + TRANSACTION + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/create_as_col_qual_list.html b/_includes/v20.2/sql/diagrams/create_as_col_qual_list.html new file mode 100644 index 00000000000..791829c3ba3 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_as_col_qual_list.html @@ -0,0 +1,17 @@ +
+ + + + +PRIMARY + + +KEY + + +FAMILY + + +family_name + +
diff --git a/_includes/v20.2/sql/diagrams/create_as_constraint_def.html b/_includes/v20.2/sql/diagrams/create_as_constraint_def.html new file mode 100644 index 00000000000..3699d2b1833 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_as_constraint_def.html @@ -0,0 +1,20 @@ +
+ + + + +PRIMARY + + +KEY + + +( + + +create_as_params + +) + + +
diff --git a/_includes/v20.2/sql/diagrams/create_changefeed.html b/_includes/v20.2/sql/diagrams/create_changefeed.html new file mode 100644 index 00000000000..82b77b8360e --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_changefeed.html @@ -0,0 +1,46 @@ +
+ + + + +CREATE + + +CHANGEFEED + + +FOR + + +TABLE + + +table_name + + +, + + +INTO + + +sink + + +WITH + + +option + + += + + +value + + +, + + + +
diff --git a/_includes/v20.2/sql/diagrams/create_database.html b/_includes/v20.2/sql/diagrams/create_database.html new file mode 100644 index 00000000000..c621b08e138 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_database.html @@ -0,0 +1,42 @@ +
+ + + + + + CREATE + + + DATABASE + + + IF + + + NOT + + + EXISTS + + + + name + + + + WITH + + + ENCODING + + + = + + + + encoding + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/create_index.html b/_includes/v20.2/sql/diagrams/create_index.html new file mode 100644 index 00000000000..efef77b3721 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_index.html @@ -0,0 +1,122 @@ +
+ + + + +CREATE + + +UNIQUE + + +INDEX + + +CONCURRENTLY + + + +opt_index_name + + + +IF + + +NOT + + +EXISTS + + + +index_name + + + +ON + + + +table_name + + + +USING + + + +name + + + +( + + + +a_expr + + + +ASC + + +DESC + + +, + + +) + + +USING + + +HASH + + +WITH + + +BUCKET_COUNT + + += + + +n_buckets + + +COVERING + + +STORING + + +INCLUDE + + +( + + + +name_list + + + +) + + + +opt_interleave + + + + +opt_partition_by + + + + +
diff --git a/_includes/v20.2/sql/diagrams/create_inverted_index.html b/_includes/v20.2/sql/diagrams/create_inverted_index.html new file mode 100644 index 00000000000..92de493da93 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_inverted_index.html @@ -0,0 +1,99 @@ +
+ + + + +CREATE + + +UNIQUE + + +INVERTED + + +INDEX + + +CONCURRENTLY + + + +opt_index_name + + + +IF + + +NOT + + +EXISTS + + + +index_name + + + +ON + + + +table_name + + + +( + + + +a_expr + + + +ASC + + +DESC + + +, + + +) + + +COVERING + + +STORING + + +INCLUDE + + +( + + + +name_list + + + +) + + + +opt_interleave + + + + +opt_partition_by + + + + +
diff --git a/_includes/v20.2/sql/diagrams/create_role.html b/_includes/v20.2/sql/diagrams/create_role.html new file mode 100644 index 00000000000..3c9c43dedf3 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_role.html @@ -0,0 +1,28 @@ +
+ + + + + + CREATE + + + ROLE + + + IF + + + NOT + + + EXISTS + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/create_sequence.html b/_includes/v20.2/sql/diagrams/create_sequence.html new file mode 100644 index 00000000000..134f4e59596 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_sequence.html @@ -0,0 +1,80 @@ +
+ + + + +CREATE + + + +opt_temp + + + +SEQUENCE + + +IF + + +NOT + + +EXISTS + + +sequence_name + + +NO + + +CYCLE + + +MINVALUE + + +MAXVALUE + + +OWNED + + +BY + + +NONE + + + +column_name + + + +INCREMENT + + +BY + + +MINVALUE + + +MAXVALUE + + +START + + +WITH + + +integer + + +VIRTUAL + + + +
diff --git a/_includes/v20.2/sql/diagrams/create_stats.html b/_includes/v20.2/sql/diagrams/create_stats.html new file mode 100644 index 00000000000..c02186ee5cb --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_stats.html @@ -0,0 +1,25 @@ +
+ + + + +CREATE + + +STATISTICS + + +statistics_name + + +opt_stats_columns + +FROM + + +create_stats_target + + +opt_as_of_clause + +
diff --git a/_includes/v20.2/sql/diagrams/create_table.html b/_includes/v20.2/sql/diagrams/create_table.html new file mode 100644 index 00000000000..5a24a98e25c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_table.html @@ -0,0 +1,71 @@ + diff --git a/_includes/v20.2/sql/diagrams/create_table_as.html b/_includes/v20.2/sql/diagrams/create_table_as.html new file mode 100644 index 00000000000..8023d7826d5 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_table_as.html @@ -0,0 +1,79 @@ + diff --git a/_includes/v20.2/sql/diagrams/create_user.html b/_includes/v20.2/sql/diagrams/create_user.html new file mode 100644 index 00000000000..1dc78bb289a --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_user.html @@ -0,0 +1,39 @@ +
+ + + + + + CREATE + + + USER + + + IF + + + NOT + + + EXISTS + + + + name + + + + WITH + + + PASSWORD + + + + password + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/create_view.html b/_includes/v20.2/sql/diagrams/create_view.html new file mode 100644 index 00000000000..fe10170bff9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/create_view.html @@ -0,0 +1,51 @@ +
+ + + + +CREATE + + + +opt_temp + + + +VIEW + + +IF + + +NOT + + +EXISTS + + + +view_name + + + +( + + + +name_list + + + +) + + +AS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/default_value_column_level.html b/_includes/v20.2/sql/diagrams/default_value_column_level.html new file mode 100644 index 00000000000..0ba9afca9c4 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/default_value_column_level.html @@ -0,0 +1,64 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + DEFAULT + + + + default_value + + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/delete.html b/_includes/v20.2/sql/diagrams/delete.html new file mode 100644 index 00000000000..746c3d41e08 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/delete.html @@ -0,0 +1,52 @@ +
+ + + + +WITH + + +RECURSIVE + + +common_table_expr + +, + + +DELETE + + +FROM + + +table_name + + +opt_index_flags + +AS + + +table_alias_name + +WHERE + + +a_expr + + +sort_clause + + +limit_clause + +RETURNING + + +target_list + +NOTHING + + +
diff --git a/_includes/v20.2/sql/diagrams/drop.html b/_includes/v20.2/sql/diagrams/drop.html new file mode 100644 index 00000000000..c6034b008aa --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop.html @@ -0,0 +1,33 @@ + \ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_column.html b/_includes/v20.2/sql/diagrams/drop_column.html new file mode 100644 index 00000000000..384f5219d9d --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_column.html @@ -0,0 +1,43 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +DROP + + +COLUMN + + +IF + + +EXISTS + + +name + + +CASCADE + + +RESTRICT + + + +
diff --git a/_includes/v20.2/sql/diagrams/drop_constraint.html b/_includes/v20.2/sql/diagrams/drop_constraint.html new file mode 100644 index 00000000000..77cea230ccd --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_constraint.html @@ -0,0 +1,45 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + + +DROP + + +CONSTRAINT + + +IF + + +EXISTS + + + +name + + + +CASCADE + + +RESTRICT + + + +
diff --git a/_includes/v20.2/sql/diagrams/drop_database.html b/_includes/v20.2/sql/diagrams/drop_database.html new file mode 100644 index 00000000000..038eb0befc1 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_database.html @@ -0,0 +1,31 @@ +
+ + + + + + DROP + + + DATABASE + + + IF + + + EXISTS + + + + name + + + + CASCADE + + + RESTRICT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_index.html b/_includes/v20.2/sql/diagrams/drop_index.html new file mode 100644 index 00000000000..3e50bf8d4b1 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_index.html @@ -0,0 +1,41 @@ +
+ + + + +DROP + + +INDEX + + +CONCURRENTLY + + +IF + + +EXISTS + + + +table_name + + + +@ + + + +index_name + + + +CASCADE + + +RESTRICT + + + +
diff --git a/_includes/v20.2/sql/diagrams/drop_role.html b/_includes/v20.2/sql/diagrams/drop_role.html new file mode 100644 index 00000000000..0037ebf56ce --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_role.html @@ -0,0 +1,25 @@ +
+ + + + + + DROP + + + ROLE + + + IF + + + EXISTS + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_sequence.html b/_includes/v20.2/sql/diagrams/drop_sequence.html new file mode 100644 index 00000000000..6507f7dec30 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_sequence.html @@ -0,0 +1,34 @@ +
+ + + + + + DROP + + + SEQUENCE + + + IF + + + EXISTS + + + + sequence_name + + + + , + + + CASCADE + + + RESTRICT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_table.html b/_includes/v20.2/sql/diagrams/drop_table.html new file mode 100644 index 00000000000..18ad4fdd502 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_table.html @@ -0,0 +1,34 @@ +
+ + + + + + DROP + + + TABLE + + + IF + + + EXISTS + + + + table_name + + + + , + + + CASCADE + + + RESTRICT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_user.html b/_includes/v20.2/sql/diagrams/drop_user.html new file mode 100644 index 00000000000..57c3db991b9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_user.html @@ -0,0 +1,28 @@ +
+ + + + + + DROP + + + USER + + + IF + + + EXISTS + + + + user_name + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/drop_view.html b/_includes/v20.2/sql/diagrams/drop_view.html new file mode 100644 index 00000000000..d95db116000 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/drop_view.html @@ -0,0 +1,34 @@ +
+ + + + + + DROP + + + VIEW + + + IF + + + EXISTS + + + + table_name + + + + , + + + CASCADE + + + RESTRICT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/experimental_audit.html b/_includes/v20.2/sql/diagrams/experimental_audit.html new file mode 100644 index 00000000000..46cc527074a --- /dev/null +++ b/_includes/v20.2/sql/diagrams/experimental_audit.html @@ -0,0 +1,39 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + + +table_name + + + +EXPERIMENTAL_AUDIT + + +SET + + +READ + + +WRITE + + +OFF + + + +
diff --git a/_includes/v20.2/sql/diagrams/explain.html b/_includes/v20.2/sql/diagrams/explain.html new file mode 100644 index 00000000000..eb8f361e704 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/explain.html @@ -0,0 +1,35 @@ +
+ + + + +EXPLAIN + + +( + + +VERBOSE + + +TYPES + + +OPT + + +DISTSQL + + +VEC + + +, + + +) + + +preparable_stmt + +
diff --git a/_includes/v20.2/sql/diagrams/explain_analyze.html b/_includes/v20.2/sql/diagrams/explain_analyze.html new file mode 100644 index 00000000000..37d76fa8351 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/explain_analyze.html @@ -0,0 +1,36 @@ +
+ + + + +EXPLAIN + + +ANALYZE + + +ANALYSE + + +( + + +DISTSQL + + +DEBUG + + +, + + +) + + + +preparable_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/export.html b/_includes/v20.2/sql/diagrams/export.html new file mode 100644 index 00000000000..05ad8e2a864 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/export.html @@ -0,0 +1,36 @@ +
+ + + + +EXPORT + + +INTO + + +CSV + + +file_location + + + +opt_with_options + + + +FROM + + +select_stmt + + +TABLE + + +table_name + + + +
diff --git a/_includes/v20.2/sql/diagrams/family_def.html b/_includes/v20.2/sql/diagrams/family_def.html new file mode 100644 index 00000000000..1dda01d9e79 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/family_def.html @@ -0,0 +1,30 @@ +
+ + + + + + FAMILY + + + + opt_family_name + + + + ( + + + + name + + + + , + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/foreign_key_column_level.html b/_includes/v20.2/sql/diagrams/foreign_key_column_level.html new file mode 100644 index 00000000000..a963e586425 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/foreign_key_column_level.html @@ -0,0 +1,75 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + REFERENCES + + + + parent_table + + + + ( + + + + ref_column_name + + + + ) + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/foreign_key_table_level.html b/_includes/v20.2/sql/diagrams/foreign_key_table_level.html new file mode 100644 index 00000000000..2eb3498af46 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/foreign_key_table_level.html @@ -0,0 +1,85 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_def + + + + , + + + CONSTRAINT + + + + name + + + + FOREIGN KEY + + + ( + + + + fk_column_name + + + + , + + + ) + + + REFERENCES + + + + parent_table + + + + ( + + + + ref_column_name + + + + , + + + ) + + + + table_constraints + + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/grammar.html b/_includes/v20.2/sql/diagrams/grammar.html new file mode 100644 index 00000000000..71d8cf930a9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/grammar.html @@ -0,0 +1,10848 @@ +
+ + +

stmt_block:

+ + + + + + + + stmt_list + + + + + +

no references


stmt_list:

+ + + + + + + + stmt + + + + ; + + + + +

referenced by: +

+


stmt:

+ + + + + + + HELPTOKEN + + + + alter_stmt + + + + + backup_stmt + + + + + cancel_stmt + + + + + copy_from_stmt + + + + + create_stmt + + + + + deallocate_stmt + + + + + delete_stmt + + + + + discard_stmt + + + + + drop_stmt + + + + + execute_stmt + + + + + explain_stmt + + + + + grant_stmt + + + + + insert_stmt + + + + + import_stmt + + + + + pause_stmt + + + + + prepare_stmt + + + + + restore_stmt + + + + + resume_stmt + + + + + revoke_stmt + + + + + savepoint_stmt + + + + + select_stmt + + + + + release_stmt + + + + + reset_stmt + + + + + set_stmt + + + + + show_stmt + + + + + transaction_stmt + + + + + truncate_stmt + + + + + update_stmt + + + + + upsert_stmt + + + + + +

referenced by: +

+


alter_stmt:

+ + + + + + + + alter_table_stmt + + + + + alter_index_stmt + + + + + alter_view_stmt + + + + + alter_database_stmt + + + + + +

referenced by: +

+


backup_stmt:

+ + + + + + + BACKUP + + + + targets + + + + TO + + + + string_or_placeholder + + + + + opt_as_of_clause + + + + + opt_incremental + + + + + opt_with_options + + + + + +

referenced by: +

+


cancel_stmt:

+ + + + + + + + cancel_job_stmt + + + + + cancel_query_stmt + + + + + +

referenced by: +

+


copy_from_stmt:

+ + + + + + + COPY + + + + qualified_name + + + + ( + + + + qualified_name_list + + + + ) + + + FROM + + + STDIN + + + + +

referenced by: +

+


create_stmt:

+ + + + + + + + create_database_stmt + + + + + create_index_stmt + + + + + create_table_stmt + + + + + create_table_as_stmt + + + + + create_user_stmt + + + + + create_view_stmt + + + + + +

referenced by: +

+


deallocate_stmt:

+ + + + + + + DEALLOCATE + + + PREPARE + + + + name + + + + ALL + + + + +

referenced by: +

+


delete_stmt:

+ + + + + + + DELETE + + + FROM + + + + relation_expr_opt_alias + + + + + where_clause + + + + + opt_limit_clause + + + + + returning_clause + + + + + +

referenced by: +

+


discard_stmt:

+ + + + + + + DISCARD + + + ALL + + + + +

referenced by: +

+


drop_stmt:

+ + + + + + + + drop_database_stmt + + + + + drop_index_stmt + + + + + drop_table_stmt + + + + + drop_view_stmt + + + + + drop_user_stmt + + + + + +

referenced by: +

+


execute_stmt:

+ + + + + + + EXECUTE + + + + name + + + + + execute_param_clause + + + + + +

referenced by: +

+


explain_stmt:

+ + + + + + + EXPLAIN + + + ( + + + + explain_option_list + + + + ) + + + + explainable_stmt + + + + + +

referenced by: +

+


grant_stmt:

+ + + + + + + GRANT + + + + privileges + + + + ON + + + + targets + + + + TO + + + + grantee_list + + + + + +

referenced by: +

+


insert_stmt:

+ + + + + + + INSERT + + + INTO + + + + insert_target + + + + + insert_rest + + + + + on_conflict + + + + + returning_clause + + + + + +

referenced by: +

+


import_stmt:

+ + + + + + + IMPORT + + + TABLE + + + + any_name + + + + CREATE + + + USING + + + + string_or_placeholder + + + + ( + + + + table_elem_list + + + + ) + + + CSV + + + DATA + + + ( + + + + string_or_placeholder_list + + + + ) + + + + opt_with_options + + + + + +

referenced by: +

+


pause_stmt:

+ + + + + + + PAUSE + + + JOB + + + + a_expr + + + + + +

referenced by: +

+


prepare_stmt:

+ + + + + + + PREPARE + + + + name + + + + + prep_type_clause + + + + AS + + + + preparable_stmt + + + + + +

referenced by: +

+


restore_stmt:

+ + + + + + + RESTORE + + + + targets + + + + FROM + + + + string_or_placeholder_list + + + + + opt_as_of_clause + + + + + opt_with_options + + + + + +

referenced by: +

+


resume_stmt:

+ + + + + + + RESUME + + + JOB + + + + a_expr + + + + + +

referenced by: +

+


revoke_stmt:

+ + + + + + + REVOKE + + + + privileges + + + + ON + + + + targets + + + + FROM + + + + grantee_list + + + + + +

referenced by: +

+


savepoint_stmt:

+ + + + + + + SAVEPOINT + + + + name + + + + + +

referenced by: +

+


select_stmt:

+ + + + + + + + select_no_parens + + + + + select_with_parens + + + + + +

referenced by: +

+


release_stmt:

+ + + + + + + RELEASE + + + + savepoint_name + + + + + +

referenced by: +

+


reset_stmt:

+ + + + + + + + reset_session_stmt + + + + + reset_csetting_stmt + + + + + +

referenced by: +

+


set_stmt:

+ + + + + + + + set_session_stmt + + + + + set_csetting_stmt + + + + + set_transaction_stmt + + + + + +

referenced by: +

+


show_stmt:

+ + + + + + + + show_backup_stmt + + + + + show_columns_stmt + + + + + show_constraints_stmt + + + + + show_create_table_stmt + + + + + show_create_view_stmt + + + + + show_csettings_stmt + + + + + show_databases_stmt + + + + + show_grants_stmt + + + + + show_indexes_stmt + + + + + show_jobs_stmt + + + + + show_queries_stmt + + + + + show_session_stmt + + + + + show_sessions_stmt + + + + + show_tables_stmt + + + + + show_trace_stmt + + + + + show_users_stmt + + + + + +

referenced by: +

+


transaction_stmt:

+ + + + + + + + begin_stmt + + + + + commit_stmt + + + + + rollback_stmt + + + + + +

referenced by: +

+


truncate_stmt:

+ + + + + + + TRUNCATE + + + + opt_table + + + + + relation_expr_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


update_stmt:

+ + + + + + + UPDATE + + + + relation_expr_opt_alias + + + + SET + + + + set_clause_list + + + + + where_clause + + + + + returning_clause + + + + + +

referenced by: +

+


upsert_stmt:

+ + + + + + + UPSERT + + + INTO + + + + insert_target + + + + + insert_rest + + + + + returning_clause + + + + + +

referenced by: +

+


alter_table_stmt:

+ + + + + + + + alter_onetable_stmt + + + + + alter_split_stmt + + + + + alter_scatter_stmt + + + + + alter_rename_table_stmt + + + + + +

referenced by: +

+


alter_index_stmt:

+ + + + + + + + alter_split_index_stmt + + + + + alter_scatter_index_stmt + + + + + alter_rename_index_stmt + + + + + +

referenced by: +

+


alter_view_stmt:

+ + + + + + + + alter_rename_view_stmt + + + + + +

referenced by: +

+


alter_database_stmt:

+ + + + + + + + alter_rename_database_stmt + + + + + +

referenced by: +

+


targets:

+ + + + + + + TABLE + + + + table_pattern_list + + + + DATABASE + + + + name_list + + + + + +

referenced by: +

+


string_or_placeholder:

+ + + + + + + + non_reserved_word_or_sconst + + + + PLACEHOLDER + + + + +

referenced by: +

+


opt_as_of_clause:

+ + + + + + + AS + + + OF + + + SYSTEM + + + TIME + + + + a_expr_const + + + + + +

referenced by: +

+


opt_incremental:

+ + + + + + + INCREMENTAL + + + FROM + + + + string_or_placeholder_list + + + + + +

referenced by: +

+


opt_with_options:

+ + + + + + + WITH + + + + kv_option_list + + + + OPTIONS + + + ( + + + + kv_option_list + + + + ) + + + + +

referenced by: +

+


cancel_job_stmt:

+ + + + + + + CANCEL + + + JOB + + + + a_expr + + + + + +

referenced by: +

+


cancel_query_stmt:

+ + + + + + + CANCEL + + + QUERY + + + + a_expr + + + + + +

referenced by: +

+


qualified_name:

+ + + + + + + + name + + + + + qname_indirection + + + + + +

referenced by: +

+


qualified_name_list:

+ + + + + + + + qualified_name + + + + , + + + + +

referenced by: +

+


create_database_stmt:

+ + + + + + + CREATE + + + DATABASE + + + IF + + + NOT + + + EXISTS + + + + name + + + + + opt_with + + + + + opt_template_clause + + + + + opt_encoding_clause + + + + + opt_lc_collate_clause + + + + + opt_lc_ctype_clause + + + + + +

referenced by: +

+


create_index_stmt:

+ + + + + + + CREATE + + + + opt_unique + + + + INDEX + + + + opt_name + + + + IF + + + NOT + + + EXISTS + + + + name + + + + ON + + + + qualified_name + + + + ( + + + + index_params + + + + ) + + + + opt_storing + + + + + opt_interleave + + + + + +

referenced by: +

+


create_table_stmt:

+ + + + + + + CREATE + + + TABLE + + + IF + + + NOT + + + EXISTS + + + + any_name + + + + ( + + + + opt_table_elem_list + + + + ) + + + + opt_interleave + + + + + +

referenced by: +

+


create_table_as_stmt:

+ + + + + + + CREATE + + + TABLE + + + IF + + + NOT + + + EXISTS + + + + any_name + + + + + opt_column_list + + + + AS + + + + select_stmt + + + + + +

referenced by: +

+


create_user_stmt:

+ + + + + + + CREATE + + + USER + + + + name + + + + + opt_password + + + + + +

referenced by: +

+


create_view_stmt:

+ + + + + + + CREATE + + + VIEW + + + + any_name + + + + + opt_column_list + + + + AS + + + + select_stmt + + + + + +

referenced by: +

+


name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + +

referenced by: +

+


relation_expr_opt_alias:

+ + + + + + + + relation_expr + + + + AS + + + + name + + + + + +

referenced by: +

+


where_clause:

+ + + + + + + WHERE + + + + a_expr + + + + + +

referenced by: +

+


opt_limit_clause:

+ + + + + + + + limit_clause + + + + + +

referenced by: +

+


returning_clause:

+ + + + + + + RETURNING + + + + target_list + + + + NOTHING + + + + +

referenced by: +

+


drop_database_stmt:

+ + + + + + + DROP + + + DATABASE + + + IF + + + EXISTS + + + + name + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_index_stmt:

+ + + + + + + DROP + + + INDEX + + + IF + + + EXISTS + + + + table_name_with_index_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_table_stmt:

+ + + + + + + DROP + + + TABLE + + + IF + + + EXISTS + + + + table_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_view_stmt:

+ + + + + + + DROP + + + VIEW + + + IF + + + EXISTS + + + + table_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_user_stmt:

+ + + + + + + DROP + + + USER + + + IF + + + EXISTS + + + + name_list + + + + + +

referenced by: +

+


execute_param_clause:

+ + + + + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


explainable_stmt:

+ + + + + + + + preparable_stmt + + + + + alter_stmt + + + + + create_stmt + + + + + drop_stmt + + + + + execute_stmt + + + + + +

referenced by: +

+


explain_option_list:

+ + + + + + + + explain_option_name + + + + , + + + + +

referenced by: +

+


privileges:

+ + + + + + + ALL + + + + privilege_list + + + + + +

referenced by: +

+


grantee_list:

+ + + + + + + + name + + + + , + + + + +

referenced by: +

+


insert_target:

+ + + + + + + + qualified_name + + + + AS + + + + name + + + + + +

referenced by: +

+


insert_rest:

+ + + + + + + ( + + + + qualified_name_list + + + + ) + + + + select_stmt + + + + DEFAULT + + + VALUES + + + + +

referenced by: +

+


on_conflict:

+ + + + + + + ON + + + CONFLICT + + + + opt_conf_expr + + + + DO + + + UPDATE + + + SET + + + + set_clause_list + + + + + where_clause + + + + NOTHING + + + + +

referenced by: +

+


any_name:

+ + + + + + + + name + + + + + attrs + + + + + +

referenced by: +

+


string_or_placeholder_list:

+ + + + + + + + string_or_placeholder + + + + , + + + + +

referenced by: +

+


table_elem_list:

+ + + + + + + + table_elem + + + + , + + + + +

referenced by: +

+


a_expr:

+ + + + + + + + c_expr + + + + + + + + - + + + ~ + + + NOT + + + + a_expr + + + + TYPECAST + + + + cast_target + + + + TYPEANNOTATE + + + + typename + + + + COLLATE + + + + unrestricted_name + + + + + + + + - + + + * + + + / + + + FLOORDIV + + + % + + + ^ + + + # + + + & + + + | + + + < + + + > + + + = + + + CONCAT + + + LSHIFT + + + RSHIFT + + + LESS_EQUALS + + + GREATER_EQUALS + + + NOT_EQUALS + + + BETWEEN + + + + opt_asymmetric + + + + SYMMETRIC + + + + b_expr + + + + AND + + + OR + + + LIKE + + + ILIKE + + + SIMILAR + + + TO + + + ~ + + + NOT_REGMATCH + + + REGIMATCH + + + NOT_REGIMATCH + + + + a_expr + + + + NOT + + + LIKE + + + ILIKE + + + SIMILAR + + + TO + + + BETWEEN + + + + opt_asymmetric + + + + SYMMETRIC + + + + b_expr + + + + AND + + + + a_expr + + + + IN + + + + in_expr + + + + IS + + + NOT + + + NAN + + + NULL + + + TRUE + + + FALSE + + + UNKNOWN + + + DISTINCT + + + FROM + + + + a_expr + + + + OF + + + ( + + + + type_list + + + + ) + + + IN + + + + in_expr + + + + + subquery_op + + + + + sub_type + + + + + d_expr + + + + + +

referenced by: +

+


prep_type_clause:

+ + + + + + + ( + + + + type_list + + + + ) + + + + +

referenced by: +

+


preparable_stmt:

+ + + + + + + + backup_stmt + + + + + cancel_stmt + + + + + delete_stmt + + + + + import_stmt + + + + + insert_stmt + + + + + pause_stmt + + + + + reset_stmt + + + + + restore_stmt + + + + + resume_stmt + + + + + select_stmt + + + + + set_session_stmt + + + + + set_csetting_stmt + + + + + show_stmt + + + + + update_stmt + + + + + upsert_stmt + + + + + +

referenced by: +

+


select_no_parens:

+ + + + + + + + simple_select + + + + + select_clause + + + + + sort_clause + + + + + opt_sort_clause + + + + + select_limit + + + + + +

referenced by: +

+


select_with_parens:

+ + + + + + + ( + + + + select_no_parens + + + + + select_with_parens + + + + ) + + + + +

referenced by: +

+


savepoint_name:

+ + + + + + + SAVEPOINT + + + + name + + + + + +

referenced by: +

+


reset_session_stmt:

+ + + + + + + RESET + + + SESSION + + + + session_var + + + + + +

referenced by: +

+


reset_csetting_stmt:

+ + + + + + + RESET + + + CLUSTER + + + SETTING + + + + var_name + + + + + +

referenced by: +

+


set_session_stmt:

+ + + + + + + SET + + + SESSION + + + + set_rest_more + + + + CHARACTERISTICS + + + AS + + + TRANSACTION + + + + transaction_iso_level + + + + + set_rest_more + + + + + +

referenced by: +

+


set_csetting_stmt:

+ + + + + + + SET + + + CLUSTER + + + SETTING + + + + var_name + + + + = + + + TO + + + + var_value + + + + + +

referenced by: +

+


set_transaction_stmt:

+ + + + + + + SET + + + SESSION + + + TRANSACTION + + + + transaction_mode_list + + + + + +

referenced by: +

+


show_backup_stmt:

+ + + + + + + SHOW + + + BACKUP + + + + string_or_placeholder + + + + + +

referenced by: +

+


show_columns_stmt:

+ + + + + + + SHOW + + + COLUMNS + + + FROM + + + + var_name + + + + + +

referenced by: +

+


show_constraints_stmt:

+ + + + + + + SHOW + + + CONSTRAINT + + + CONSTRAINTS + + + FROM + + + + var_name + + + + + +

referenced by: +

+


show_create_table_stmt:

+ + + + + + + SHOW + + + CREATE + + + TABLE + + + + var_name + + + + + +

referenced by: +

+


show_create_view_stmt:

+ + + + + + + SHOW + + + CREATE + + + VIEW + + + + var_name + + + + + +

referenced by: +

+


show_csettings_stmt:

+ + + + + + + SHOW + + + CLUSTER + + + SETTING + + + + any_name + + + + ALL + + + ALL + + + CLUSTER + + + SETTINGS + + + + +

referenced by: +

+


show_databases_stmt:

+ + + + + + + SHOW + + + DATABASES + + + + +

referenced by: +

+


show_grants_stmt:

+ + + + + + + SHOW + + + GRANTS + + + + on_privilege_target_clause + + + + + for_grantee_clause + + + + + +

referenced by: +

+


show_indexes_stmt:

+ + + + + + + SHOW + + + INDEX + + + INDEXES + + + KEYS + + + FROM + + + + var_name + + + + + +

referenced by: +

+


show_jobs_stmt:

+ + + + + + + SHOW + + + JOBS + + + + +

referenced by: +

+


show_queries_stmt:

+ + + + + + + SHOW + + + CLUSTER + + + LOCAL + + + QUERIES + + + + +

referenced by: +

+


show_session_stmt:

+ + + + + + + SHOW + + + SESSION + + + + session_var + + + + + +

referenced by: +

+


show_sessions_stmt:

+ + + + + + + SHOW + + + CLUSTER + + + LOCAL + + + SESSIONS + + + + +

referenced by: +

+


show_tables_stmt:

+ + + + + + + SHOW + + + TABLES + + + FROM + + + + name + + + + + +

referenced by: +

+


show_trace_stmt:

+ + + + + + + SHOW + + + KV + + + TRACE + + + FOR + + + SESSION + + + + explainable_stmt + + + + + +

referenced by: +

+


show_users_stmt:

+ + + + + + + SHOW + + + USERS + + + + +

referenced by: +

+


begin_stmt:

+ + + + + + + BEGIN + + + + opt_transaction + + + + START + + + TRANSACTION + + + + begin_transaction + + + + + +

referenced by: +

+


commit_stmt:

+ + + + + + + COMMIT + + + END + + + + opt_transaction + + + + + +

referenced by: +

+


rollback_stmt:

+ + + + + + + ROLLBACK + + + + opt_to_savepoint + + + + + +

referenced by: +

+


opt_table:

+ + + + + + + TABLE + + + + +

referenced by: +

+


relation_expr_list:

+ + + + + + + + relation_expr + + + + , + + + + +

referenced by: +

+


opt_drop_behavior:

+ + + + + + + CASCADE + + + RESTRICT + + + + +

referenced by: +

+


set_clause_list:

+ + + + + + + + set_clause + + + + , + + + + +

referenced by: +

+


alter_onetable_stmt:

+ + + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + relation_expr + + + + + alter_table_cmds + + + + + +

referenced by: +

+


alter_split_stmt:

+ + + + + + + ALTER + + + TABLE + + + + qualified_name + + + + SPLIT + + + AT + + + + select_stmt + + + + + +

referenced by: +

+


alter_scatter_stmt:

+ + + + + + + ALTER + + + TABLE + + + + qualified_name + + + + SCATTER + + + FROM + + + ( + + + + expr_list + + + + ) + + + TO + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


alter_rename_table_stmt:

+ + + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + relation_expr + + + + RENAME + + + TO + + + + qualified_name + + + + + opt_column + + + + + name + + + + TO + + + + name + + + + + +

referenced by: +

+


alter_split_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_name_with_index + + + + SPLIT + + + AT + + + + select_stmt + + + + + +

referenced by: +

+


alter_scatter_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_name_with_index + + + + SCATTER + + + FROM + + + ( + + + + expr_list + + + + ) + + + TO + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


alter_rename_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + IF + + + EXISTS + + + + table_name_with_index + + + + RENAME + + + TO + + + + name + + + + + +

referenced by: +

+


alter_rename_view_stmt:

+ + + + + + + ALTER + + + VIEW + + + IF + + + EXISTS + + + + relation_expr + + + + RENAME + + + TO + + + + qualified_name + + + + + +

referenced by: +

+


alter_rename_database_stmt:

+ + + + + + + ALTER + + + DATABASE + + + + name + + + + RENAME + + + TO + + + + name + + + + + +

referenced by: +

+


table_pattern_list:

+ + + + + + + + table_pattern + + + + , + + + + +

referenced by: +

+


name_list:

+ + + + + + + + name + + + + , + + + + +

referenced by: +

+


non_reserved_word_or_sconst:

+ + + + + + + + non_reserved_word + + + + SCONST + + + + +

referenced by: +

+


a_expr_const:

+ + + + + + + ICONST + + + FCONST + + + + const_typename + + + + SCONST + + + BCONST + + + + interval + + + + TRUE + + + FALSE + + + NULL + + + + +

referenced by: +

+


kv_option_list:

+ + + + + + + + kv_option + + + + , + + + + +

referenced by: +

+


qname_indirection:

+ + + + + + + + name_indirection_elem + + + + + +

referenced by: +

+


opt_with:

+ + + + + + + WITH + + + + +

referenced by: +

+


opt_template_clause:

+ + + + + + + TEMPLATE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_encoding_clause:

+ + + + + + + ENCODING + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_lc_collate_clause:

+ + + + + + + LC_COLLATE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_lc_ctype_clause:

+ + + + + + + LC_CTYPE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_unique:

+ + + + + + + UNIQUE + + + + +

referenced by: +

+


opt_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


index_params:

+ + + + + + + + index_elem + + + + , + + + + +

referenced by: +

+


opt_storing:

+ + + + + + + + storing + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_interleave:

+ + + + + + + INTERLEAVE + + + IN + + + PARENT + + + + qualified_name + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_table_elem_list:

+ + + + + + + + table_elem_list + + + + + +

referenced by: +

+


opt_column_list:

+ + + + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_password:

+ + + + + + + + opt_with + + + + PASSWORD + + + SCONST + + + + +

referenced by: +

+


unreserved_keyword:

+ + + + + + + ACTION + + + ADD + + + ALTER + + + AT + + + BACKUP + + + BEGIN + + + BLOB + + + BY + + + CANCEL + + + CASCADE + + + CLUSTER + + + COLUMNS + + + COMMIT + + + COMMITTED + + + CONFLICT + + + CONSTRAINTS + + + COPY + + + COVERING + + + CSV + + + CUBE + + + CURRENT + + + CYCLE + + + DATA + + + DATABASE + + + DATABASES + + + DAY + + + DEALLOCATE + + + DELETE + + + DISCARD + + + DOUBLE + + + DROP + + + ENCODING + + + EXECUTE + + + EXPERIMENTAL_FINGERPRINTS + + + EXPLAIN + + + FILTER + + + FIRST + + + FOLLOWING + + + FORCE_INDEX + + + GRANTS + + + HIGH + + + HOUR + + + IMPORT + + + INCREMENTAL + + + INDEXES + + + INSERT + + + INT2VECTOR + + + INTERLEAVE + + + ISOLATION + + + JOB + + + JOBS + + + KEY + + + KEYS + + + KV + + + LC_COLLATE + + + LC_CTYPE + + + LEVEL + + + LOCAL + + + LOW + + + MATCH + + + MINUTE + + + MONTH + + + NAMES + + + NAN + + + NEXT + + + NO + + + NORMAL + + + NO_INDEX_JOIN + + + NULLS + + + OF + + + OFF + + + OID + + + OPTIONS + + + ORDINALITY + + + OVER + + + PARENT + + + PARTIAL + + + PARTITION + + + PASSWORD + + + PAUSE + + + PLANS + + + PRECEDING + + + PREPARE + + + PRIORITY + + + QUERIES + + + QUERY + + + RANGE + + + READ + + + RECURSIVE + + + REF + + + REGCLASS + + + REGPROC + + + REGPROCEDURE + + + REGNAMESPACE + + + REGTYPE + + + RELEASE + + + RENAME + + + REPEATABLE + + + RESET + + + RESTORE + + + RESTRICT + + + RESUME + + + REVOKE + + + ROLLBACK + + + ROLLUP + + + ROWS + + + SETTING + + + SETTINGS + + + STATUS + + + SAVEPOINT + + + SCATTER + + + SEARCH + + + SECOND + + + SERIALIZABLE + + + SEQUENCES + + + SESSION + + + SESSIONS + + + SET + + + SHOW + + + SIMPLE + + + SNAPSHOT + + + SQL + + + START + + + STDIN + + + STORE + + + STORING + + + STRICT + + + SPLIT + + + SYSTEM + + + TABLES + + + TEMP + + + TEMPLATE + + + TEMPORARY + + + TESTING_RANGES + + + TESTING_RELOCATE + + + TEXT + + + TRACE + + + TRANSACTION + + + TRUNCATE + + + TYPE + + + UNBOUNDED + + + UNCOMMITTED + + + UNKNOWN + + + UPDATE + + + UPSERT + + + USE + + + USERS + + + VALID + + + VALIDATE + + + VALUE + + + VARYING + + + WITHIN + + + WITHOUT + + + WRITE + + + YEAR + + + ZONE + + + + +

referenced by: +

+


col_name_keyword:

+ + + + + + + ANNOTATE_TYPE + + + BETWEEN + + + BIGINT + + + BIGSERIAL + + + BIT + + + BOOL + + + BOOLEAN + + + BYTEA + + + BYTES + + + CHAR + + + CHARACTER + + + CHARACTERISTICS + + + COALESCE + + + DATE + + + DEC + + + DECIMAL + + + EXISTS + + + EXTRACT + + + EXTRACT_DURATION + + + FLOAT + + + FLOAT4 + + + FLOAT8 + + + GREATEST + + + GROUPING + + + IF + + + IFNULL + + + INT + + + INT2 + + + INT4 + + + INT8 + + + INT64 + + + INTEGER + + + INTERVAL + + + LEAST + + + NAME + + + NULLIF + + + NUMERIC + + + OUT + + + OVERLAY + + + POSITION + + + PRECISION + + + REAL + + + ROW + + + SERIAL + + + SMALLINT + + + SMALLSERIAL + + + STRING + + + SUBSTRING + + + TIME + + + TIMESTAMP + + + TIMESTAMPTZ + + + TREAT + + + TRIM + + + UUID + + + VALUES + + + VARCHAR + + + + +

referenced by: +

+


relation_expr:

+ + + + + + + + qualified_name + + + + * + + + ONLY + + + + qualified_name + + + + ( + + + + qualified_name + + + + ) + + + + +

referenced by: +

+


limit_clause:

+ + + + + + + LIMIT + + + + select_limit_value + + + + FETCH + + + + first_or_next + + + + + opt_select_fetch_first_value + + + + + row_or_rows + + + + ONLY + + + + +

referenced by: +

+


target_list:

+ + + + + + + + target_elem + + + + , + + + + +

referenced by: +

+


table_name_with_index_list:

+ + + + + + + + table_name_with_index + + + + , + + + + +

referenced by: +

+


table_name_list:

+ + + + + + + + any_name + + + + , + + + + +

referenced by: +

+


expr_list:

+ + + + + + + + a_expr + + + + , + + + + +

referenced by: +

+


explain_option_name:

+ + + + + + + + non_reserved_word + + + + + +

referenced by: +

+


privilege_list:

+ + + + + + + + privilege + + + + , + + + + +

referenced by: +

+


opt_conf_expr:

+ + + + + + + ( + + + + name_list + + + + ) + + + + where_clause + + + + + +

referenced by: +

+


attrs:

+ + + + + + + . + + + + unrestricted_name + + + + + +

referenced by: +

+


table_elem:

+ + + + + + + + column_def + + + + + index_def + + + + + family_def + + + + + table_constraint + + + + + +

referenced by: +

+


c_expr:

+ + + + + + + + d_expr + + + + + array_subscripts + + + + + case_expr + + + + EXISTS + + + + select_with_parens + + + + + +

referenced by: +

+


cast_target:

+ + + + + + + + typename + + + + + postgres_oid + + + + + +

referenced by: +

+


typename:

+ + + + + + + + simple_typename + + + + + opt_array_bounds + + + + ARRAY + + + + +

referenced by: +

+


unrestricted_name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + type_func_name_keyword + + + + + reserved_keyword + + + + + +

referenced by: +

+


type_list:

+ + + + + + + + typename + + + + , + + + + +

referenced by: +

+


opt_asymmetric:

+ + + + + + + ASYMMETRIC + + + + +

referenced by: +

+


b_expr:

+ + + + + + + + c_expr + + + + + + + + - + + + ~ + + + + b_expr + + + + TYPECAST + + + + cast_target + + + + TYPEANNOTATE + + + + typename + + + + + + + + - + + + * + + + / + + + FLOORDIV + + + % + + + ^ + + + # + + + & + + + | + + + < + + + > + + + = + + + CONCAT + + + LSHIFT + + + RSHIFT + + + LESS_EQUALS + + + GREATER_EQUALS + + + NOT_EQUALS + + + + b_expr + + + + IS + + + NOT + + + DISTINCT + + + FROM + + + + b_expr + + + + OF + + + ( + + + + type_list + + + + ) + + + + +

referenced by: +

+


in_expr:

+ + + + + + + + select_with_parens + + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


subquery_op:

+ + + + + + + + math_op + + + + NOT + + + LIKE + + + ILIKE + + + + +

referenced by: +

+


sub_type:

+ + + + + + + ANY + + + SOME + + + ALL + + + + +

referenced by: +

+


d_expr:

+ + + + + + + + qualified_name + + + + + a_expr_const + + + + @ + + + ICONST + + + PLACEHOLDER + + + ( + + + + a_expr + + + + ) + + + + func_expr + + + + + select_with_parens + + + + ARRAY + + + + select_with_parens + + + + + array_expr + + + + + explicit_row + + + + + implicit_row + + + + + +

referenced by: +

+


simple_select:

+ + + + + + + + simple_select_clause + + + + + values_clause + + + + + table_clause + + + + + set_operation + + + + + +

referenced by: +

+


select_clause:

+ + + + + + + + simple_select + + + + + select_with_parens + + + + + +

referenced by: +

+


sort_clause:

+ + + + + + + ORDER + + + BY + + + + sortby_list + + + + + +

referenced by: +

+


opt_sort_clause:

+ + + + + + + + sort_clause + + + + + +

referenced by: +

+


select_limit:

+ + + + + + + + limit_clause + + + + + offset_clause + + + + + offset_clause + + + + + limit_clause + + + + + +

referenced by: +

+


session_var:

+ + + + + + + identifier + + + ALL + + + DATABASE + + + NAMES + + + SESSION_USER + + + TIME + + + ZONE + + + + +

referenced by: +

+


var_name:

+ + + + + + + + any_name + + + + + +

referenced by: +

+


set_rest_more:

+ + + + + + + + generic_set + + + + + +

referenced by: +

+


transaction_iso_level:

+ + + + + + + ISOLATION + + + LEVEL + + + + iso_level + + + + + +

referenced by: +

+


var_value:

+ + + + + + + + ctext_expr + + + + ON + + + + +

referenced by: +

+


transaction_mode_list:

+ + + + + + + + transaction_mode + + + + , + + + + +

referenced by: +

+


on_privilege_target_clause:

+ + + + + + + ON + + + + targets + + + + + +

referenced by: +

+


for_grantee_clause:

+ + + + + + + FOR + + + + grantee_list + + + + + +

referenced by: +

+


opt_transaction:

+ + + + + + + TRANSACTION + + + + +

referenced by: +

+


begin_transaction:

+ + + + + + + + transaction_mode_list + + + + + +

referenced by: +

+


opt_to_savepoint:

+ + + + + + + TRANSACTION + + + TO + + + + savepoint_name + + + + + +

referenced by: +

+


set_clause:

+ + + + + + + + single_set_clause + + + + + multiple_set_clause + + + + + +

referenced by: +

+


alter_table_cmds:

+ + + + + + + + alter_table_cmd + + + + , + + + + +

referenced by: +

+


opt_column:

+ + + + + + + COLUMN + + + + +

referenced by: +

+


table_name_with_index:

+ + + + + + + + qualified_name + + + + @ + + + + unrestricted_name + + + + + +

referenced by: +

+


table_pattern:

+ + + + + + + + name + + + + + name_indirection + + + + + glob_indirection + + + + * + + + + +

referenced by: +

+


non_reserved_word:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + type_func_name_keyword + + + + + +

referenced by: +

+


const_typename:

+ + + + + + + + numeric + + + + + const_bit + + + + + const_character + + + + + const_datetime + + + + + +

referenced by: +

+


interval:

+ + + + + + + INTERVAL + + + SCONST + + + + opt_interval + + + + + +

referenced by: +

+


kv_option:

+ + + + + + + + name + + + + SCONST + + + = + + + + string_or_placeholder + + + + + +

referenced by: +

+


name_indirection_elem:

+ + + + + + + + glob_indirection + + + + + name_indirection + + + + + +

referenced by: +

+


opt_equal:

+ + + + + + + = + + + + +

referenced by: +

+


index_elem:

+ + + + + + + + name + + + + + opt_asc_desc + + + + + +

referenced by: +

+


storing:

+ + + + + + + COVERING + + + STORING + + + + +

referenced by: +

+


select_limit_value:

+ + + + + + + + a_expr + + + + ALL + + + + +

referenced by: +

+


first_or_next:

+ + + + + + + FIRST + + + NEXT + + + + +

referenced by: +

+


opt_select_fetch_first_value:

+ + + + + + + + signed_iconst + + + + ( + + + + a_expr + + + + ) + + + + +

referenced by: +

+


row_or_rows:

+ + + + + + + ROW + + + ROWS + + + + +

referenced by: +

+


target_elem:

+ + + + + + + + a_expr + + + + AS + + + + unrestricted_name + + + + identifier + + + * + + + + +

referenced by: +

+


privilege:

+ + + + + + + CREATE + + + DROP + + + GRANT + + + SELECT + + + INSERT + + + DELETE + + + UPDATE + + + + +

referenced by: +

+


column_def:

+ + + + + + + + name + + + + + typename + + + + + col_qual_list + + + + + +

referenced by: +

+


index_def:

+ + + + + + + UNIQUE + + + INDEX + + + + opt_name + + + + ( + + + + index_params + + + + ) + + + + opt_storing + + + + + opt_interleave + + + + + +

referenced by: +

+


family_def:

+ + + + + + + FAMILY + + + + opt_name + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


table_constraint:

+ + + + + + + CONSTRAINT + + + + name + + + + + constraint_elem + + + + + +

referenced by: +

+


array_subscripts:

+ + + + + + + + array_subscript + + + + + +

referenced by: +

+


case_expr:

+ + + + + + + CASE + + + + case_arg + + + + + when_clause_list + + + + + case_default + + + + END + + + + +

referenced by: +

+


postgres_oid:

+ + + + + + + REGPROC + + + REGPROCEDURE + + + REGCLASS + + + REGTYPE + + + REGNAMESPACE + + + + +

referenced by: +

+


simple_typename:

+ + + + + + + + numeric + + + + + bit + + + + + character + + + + + const_datetime + + + + INTERVAL + + + + opt_interval + + + + BLOB + + + BYTES + + + BYTEA + + + TEXT + + + NAME + + + SERIAL + + + SMALLSERIAL + + + UUID + + + BIGSERIAL + + + OID + + + INT2VECTOR + + + + +

referenced by: +

+


opt_array_bounds:

+ + + + + + + [ + + + ] + + + + +

referenced by: +

+


type_func_name_keyword:

+ + + + + + + COLLATION + + + CROSS + + + FULL + + + INNER + + + ILIKE + + + IS + + + JOIN + + + LEFT + + + LIKE + + + NATURAL + + + OUTER + + + OVERLAPS + + + RIGHT + + + SIMILAR + + + + +

referenced by: +

+


reserved_keyword:

+ + + + + + + ALL + + + ANALYSE + + + ANALYZE + + + AND + + + ANY + + + ARRAY + + + AS + + + ASC + + + ASYMMETRIC + + + BOTH + + + CASE + + + CAST + + + CHECK + + + COLLATE + + + COLUMN + + + CONSTRAINT + + + CREATE + + + CURRENT_CATALOG + + + CURRENT_DATE + + + CURRENT_ROLE + + + CURRENT_SCHEMA + + + CURRENT_TIME + + + CURRENT_TIMESTAMP + + + CURRENT_USER + + + DEFAULT + + + DEFERRABLE + + + DESC + + + DISTINCT + + + DO + + + ELSE + + + END + + + EXCEPT + + + FALSE + + + FAMILY + + + FETCH + + + FOR + + + FOREIGN + + + FROM + + + GRANT + + + GROUP + + + HAVING + + + IN + + + INDEX + + + INITIALLY + + + INTERSECT + + + INTO + + + LATERAL + + + LEADING + + + LIMIT + + + LOCALTIME + + + LOCALTIMESTAMP + + + NOT + + + NOTHING + + + NULL + + + OFFSET + + + ON + + + ONLY + + + OR + + + ORDER + + + PLACING + + + PRIMARY + + + REFERENCES + + + RETURNING + + + SELECT + + + SESSION_USER + + + SOME + + + SYMMETRIC + + + TABLE + + + THEN + + + TO + + + TRAILING + + + TRUE + + + UNION + + + UNIQUE + + + USER + + + USING + + + VARIADIC + + + VIEW + + + WHEN + + + WHERE + + + WINDOW + + + WITH + + + + +

referenced by: +

+


math_op:

+ + + + + + + + + + + - + + + * + + + / + + + FLOORDIV + + + % + + + & + + + | + + + ^ + + + # + + + < + + + > + + + = + + + LESS_EQUALS + + + GREATER_EQUALS + + + NOT_EQUALS + + + + +

referenced by: +

+


func_expr:

+ + + + + + + + func_application + + + + + filter_clause + + + + + over_clause + + + + + func_expr_common_subexpr + + + + + +

referenced by: +

+


array_expr:

+ + + + + + + [ + + + + expr_list + + + + + array_expr_list + + + + ] + + + + +

referenced by: +

+


explicit_row:

+ + + + + + + ROW + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


implicit_row:

+ + + + + + + ( + + + + expr_list + + + + , + + + + a_expr + + + + ) + + + + +

referenced by: +

+


simple_select_clause:

+ + + + + + + SELECT + + + + opt_all_clause + + + + DISTINCT + + + + target_list + + + + + from_clause + + + + + where_clause + + + + + group_clause + + + + + having_clause + + + + + window_clause + + + + + +

referenced by: +

+


values_clause:

+ + + + + + + VALUES + + + + ctext_row + + + + , + + + + +

referenced by: +

+


table_clause:

+ + + + + + + TABLE + + + + table_ref + + + + + +

referenced by: +

+


set_operation:

+ + + + + + + + select_clause + + + + UNION + + + INTERSECT + + + EXCEPT + + + + all_or_distinct + + + + + select_clause + + + + + +

referenced by: +

+


sortby_list:

+ + + + + + + + sortby + + + + , + + + + +

referenced by: +

+


offset_clause:

+ + + + + + + OFFSET + + + + a_expr + + + + + c_expr + + + + + row_or_rows + + + + + +

referenced by: +

+


generic_set:

+ + + + + + + + var_name + + + + TO + + + = + + + + var_list + + + + + +

referenced by: +

+


iso_level:

+ + + + + + + READ + + + UNCOMMITTED + + + COMMITTED + + + SNAPSHOT + + + REPEATABLE + + + READ + + + SERIALIZABLE + + + + +

referenced by: +

+


ctext_expr:

+ + + + + + + + a_expr + + + + DEFAULT + + + + +

referenced by: +

+


transaction_mode:

+ + + + + + + + transaction_iso_level + + + + + transaction_user_priority + + + + + transaction_read_mode + + + + + +

referenced by: +

+


single_set_clause:

+ + + + + + + + qualified_name + + + + = + + + + ctext_expr + + + + + +

referenced by: +

+


multiple_set_clause:

+ + + + + + + ( + + + + qualified_name_list + + + + ) + + + = + + + + ctext_row + + + + + select_with_parens + + + + + +

referenced by: +

+


alter_table_cmd:

+ + + + + + + ADD + + + COLUMN + + + IF + + + NOT + + + EXISTS + + + + column_def + + + + + table_constraint + + + + + opt_validate_behavior + + + + ALTER + + + + opt_column + + + + + name + + + + + alter_column_default + + + + DROP + + + NOT + + + NULL + + + DROP + + + + opt_column + + + + CONSTRAINT + + + IF + + + EXISTS + + + + name + + + + + opt_drop_behavior + + + + VALIDATE + + + CONSTRAINT + + + + name + + + + + +

referenced by: +

+


name_indirection:

+ + + + + + + . + + + + unrestricted_name + + + + + +

referenced by: +

+


glob_indirection:

+ + + + + + + . + + + * + + + + +

referenced by: +

+


numeric:

+ + + + + + + INT + + + INT2 + + + INT4 + + + INT8 + + + INT64 + + + INTEGER + + + SMALLINT + + + BIGINT + + + REAL + + + FLOAT4 + + + FLOAT8 + + + FLOAT + + + + opt_float + + + + DOUBLE + + + PRECISION + + + DECIMAL + + + DEC + + + NUMERIC + + + + opt_numeric_modifiers + + + + BOOLEAN + + + BOOL + + + + +

referenced by: +

+


const_bit:

+ + + + + + + + bit_with_length + + + + + bit_without_length + + + + + +

referenced by: +

+


const_character:

+ + + + + + + + character_with_length + + + + + character_without_length + + + + + +

referenced by: +

+


const_datetime:

+ + + + + + + DATE + + + TIMESTAMP + + + WITHOUT + + + WITH + + + TIME + + + ZONE + + + TIMESTAMPTZ + + + + +

referenced by: +

+


opt_interval:

+ + + + + + + YEAR + + + TO + + + MONTH + + + MONTH + + + DAY + + + TO + + + HOUR + + + MINUTE + + + SECOND + + + HOUR + + + TO + + + MINUTE + + + SECOND + + + MINUTE + + + TO + + + SECOND + + + SECOND + + + + +

referenced by: +

+


opt_asc_desc:

+ + + + + + + ASC + + + DESC + + + + +

referenced by: +

+


signed_iconst:

+ + + + + + + + + + + - + + + ICONST + + + + +

referenced by: +

+


col_qual_list:

+ + + + + + + + col_qualification + + + + + +

referenced by: +

+


constraint_elem:

+ + + + + + + CHECK + + + ( + + + + a_expr + + + + PRIMARY + + + KEY + + + ( + + + + index_params + + + + ) + + + UNIQUE + + + ( + + + + index_params + + + + ) + + + + opt_storing + + + + + opt_interleave + + + + FOREIGN + + + KEY + + + ( + + + + name_list + + + + ) + + + REFERENCES + + + + qualified_name + + + + + opt_column_list + + + + + key_actions + + + + + +

referenced by: +

+


array_subscript:

+ + + + + + + [ + + + + a_expr + + + + + opt_slice_bound + + + + : + + + + opt_slice_bound + + + + ] + + + + +

referenced by: +

+


case_arg:

+ + + + + + + + a_expr + + + + + +

referenced by: +

+


when_clause_list:

+ + + + + + + + when_clause + + + + + +

referenced by: +

+


case_default:

+ + + + + + + ELSE + + + + a_expr + + + + + +

referenced by: +

+


bit:

+ + + + + + + + bit_with_length + + + + + bit_without_length + + + + + +

referenced by: +

+


character:

+ + + + + + + + character_with_length + + + + + character_without_length + + + + + +

referenced by: +

+


func_application:

+ + + + + + + + func_name + + + + ( + + + ALL + + + DISTINCT + + + + expr_list + + + + + opt_sort_clause + + + + * + + + ) + + + + +

referenced by: +

+


filter_clause:

+ + + + + + + FILTER + + + ( + + + WHERE + + + + a_expr + + + + ) + + + + +

referenced by: +

+


over_clause:

+ + + + + + + OVER + + + + window_specification + + + + + name + + + + + +

referenced by: +

+


func_expr_common_subexpr:

+ + + + + + + CURRENT_DATE + + + CURRENT_SCHEMA + + + CURRENT_TIMESTAMP + + + CURRENT_USER + + + ( + + + ) + + + SESSION_USER + + + USER + + + CAST + + + ( + + + + a_expr + + + + AS + + + + cast_target + + + + ANNOTATE_TYPE + + + ( + + + + a_expr + + + + , + + + + typename + + + + EXTRACT + + + EXTRACT_DURATION + + + ( + + + + extract_list + + + + OVERLAY + + + ( + + + + overlay_list + + + + POSITION + + + ( + + + + position_list + + + + SUBSTRING + + + ( + + + + substr_list + + + + IF + + + ( + + + + a_expr + + + + , + + + NULLIF + + + IFNULL + + + ( + + + + a_expr + + + + , + + + + a_expr + + + + COALESCE + + + GREATEST + + + LEAST + + + ( + + + + expr_list + + + + TRIM + + + ( + + + BOTH + + + LEADING + + + TRAILING + + + + trim_list + + + + ) + + + + +

referenced by: +

+


array_expr_list:

+ + + + + + + + array_expr + + + + , + + + + +

referenced by: +

+


opt_all_clause:

+ + + + + + + ALL + + + + +

referenced by: +

+


from_clause:

+ + + + + + + FROM + + + + from_list + + + + + opt_as_of_clause + + + + + +

referenced by: +

+


group_clause:

+ + + + + + + GROUP + + + BY + + + + expr_list + + + + + +

referenced by: +

+


having_clause:

+ + + + + + + HAVING + + + + a_expr + + + + + +

referenced by: +

+


window_clause:

+ + + + + + + WINDOW + + + + window_definition_list + + + + + +

referenced by: +

+


ctext_row:

+ + + + + + + ( + + + + ctext_expr_list + + + + ) + + + + +

referenced by: +

+


table_ref:

+ + + + + + + + relation_expr + + + + + opt_index_hints + + + + + qualified_name + + + + ( + + + + expr_list + + + + ) + + + + select_with_parens + + + + [ + + + + explainable_stmt + + + + ] + + + + opt_ordinality + + + + + opt_alias_clause + + + + + joined_table + + + + ( + + + + joined_table + + + + ) + + + + opt_ordinality + + + + + alias_clause + + + + + +

referenced by: +

+


all_or_distinct:

+ + + + + + + ALL + + + DISTINCT + + + + +

referenced by: +

+


sortby:

+ + + + + + + + a_expr + + + + PRIMARY + + + KEY + + + + qualified_name + + + + INDEX + + + + qualified_name + + + + @ + + + + unrestricted_name + + + + + opt_asc_desc + + + + + +

referenced by: +

+


var_list:

+ + + + + + + + var_value + + + + , + + + + +

referenced by: +

+


transaction_user_priority:

+ + + + + + + PRIORITY + + + + user_priority + + + + + +

referenced by: +

+


transaction_read_mode:

+ + + + + + + READ + + + ONLY + + + WRITE + + + + +

referenced by: +

+


alter_column_default:

+ + + + + + + SET + + + DEFAULT + + + + a_expr + + + + DROP + + + DEFAULT + + + + +

referenced by: +

+


opt_validate_behavior:

+ + + + + + + NOT + + + VALID + + + + +

referenced by: +

+


opt_float:

+ + + + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


opt_numeric_modifiers:

+ + + + + + + ( + + + ICONST + + + , + + + ICONST + + + ) + + + + +

referenced by: +

+


bit_with_length:

+ + + + + + + BIT + + + + opt_varying + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


bit_without_length:

+ + + + + + + BIT + + + + opt_varying + + + + + +

referenced by: +

+


character_with_length:

+ + + + + + + + character_base + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


character_without_length:

+ + + + + + + + character_base + + + + + +

referenced by: +

+


col_qualification:

+ + + + + + + CONSTRAINT + + + + name + + + + + col_qualification_elem + + + + COLLATE + + + + unrestricted_name + + + + FAMILY + + + + name + + + + CREATE + + + FAMILY + + + + opt_name + + + + IF + + + NOT + + + EXISTS + + + FAMILY + + + + name + + + + + +

referenced by: +

+


key_actions:

+ + + + + + + + key_update + + + + + key_delete + + + + + key_delete + + + + + key_update + + + + + +

referenced by: +

+


opt_slice_bound:

+ + + + + + + + a_expr + + + + + +

referenced by: +

+


when_clause:

+ + + + + + + WHEN + + + + a_expr + + + + THEN + + + + a_expr + + + + + +

referenced by: +

+


func_name:

+ + + + + + + + type_function_name + + + + + name + + + + + qname_indirection + + + + + +

referenced by: +

+


window_specification:

+ + + + + + + ( + + + + opt_existing_window_name + + + + + opt_partition_clause + + + + + opt_sort_clause + + + + ) + + + + +

referenced by: +

+


extract_list:

+ + + + + + + + extract_arg + + + + FROM + + + + a_expr + + + + + expr_list + + + + + +

referenced by: +

+


overlay_list:

+ + + + + + + + a_expr + + + + + overlay_placing + + + + + substr_from + + + + + substr_for + + + + + expr_list + + + + + +

referenced by: +

+


position_list:

+ + + + + + + + b_expr + + + + IN + + + + b_expr + + + + + +

referenced by: +

+


substr_list:

+ + + + + + + + a_expr + + + + + substr_from + + + + + substr_for + + + + + substr_for + + + + + substr_from + + + + + expr_list + + + + + +

referenced by: +

+


trim_list:

+ + + + + + + + a_expr + + + + FROM + + + + expr_list + + + + + +

referenced by: +

+


from_list:

+ + + + + + + + table_ref + + + + , + + + + +

referenced by: +

+


window_definition_list:

+ + + + + + + + window_definition + + + + , + + + + +

referenced by: +

+


ctext_expr_list:

+ + + + + + + + ctext_expr + + + + , + + + + +

referenced by: +

+


opt_index_hints:

+ + + + + + + @ + + + + unrestricted_name + + + + [ + + + ICONST + + + ] + + + { + + + + index_hints_param_list + + + + } + + + + +

referenced by: +

+


opt_ordinality:

+ + + + + + + WITH + + + ORDINALITY + + + + +

referenced by: +

+


opt_alias_clause:

+ + + + + + + + alias_clause + + + + + +

referenced by: +

+


joined_table:

+ + + + + + + ( + + + + joined_table + + + + ) + + + + table_ref + + + + CROSS + + + NATURAL + + + + join_type + + + + JOIN + + + + table_ref + + + + + join_type + + + + JOIN + + + + table_ref + + + + + join_qual + + + + + +

referenced by: +

+


alias_clause:

+ + + + + + + AS + + + + name + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


user_priority:

+ + + + + + + LOW + + + NORMAL + + + HIGH + + + + +

referenced by: +

+


opt_varying:

+ + + + + + + VARYING + + + + +

referenced by: +

+


character_base:

+ + + + + + + CHARACTER + + + CHAR + + + + opt_varying + + + + VARCHAR + + + STRING + + + + +

referenced by: +

+


col_qualification_elem:

+ + + + + + + NOT + + + NULL + + + UNIQUE + + + PRIMARY + + + KEY + + + CHECK + + + ( + + + + a_expr + + + + ) + + + DEFAULT + + + + b_expr + + + + REFERENCES + + + + qualified_name + + + + + opt_name_parens + + + + + key_actions + + + + + +

referenced by: +

+


key_update:

+ + + + + + + ON + + + UPDATE + + + RESTRICT + + + + +

referenced by: +

+


key_delete:

+ + + + + + + ON + + + DELETE + + + RESTRICT + + + + +

referenced by: +

+


type_function_name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + type_func_name_keyword + + + + + +

referenced by: +

+


opt_existing_window_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


opt_partition_clause:

+ + + + + + + PARTITION + + + BY + + + + expr_list + + + + + +

referenced by: +

+


extract_arg:

+ + + + + + + identifier + + + YEAR + + + MONTH + + + DAY + + + HOUR + + + MINUTE + + + SECOND + + + + +

referenced by: +

+


overlay_placing:

+ + + + + + + PLACING + + + + a_expr + + + + + +

referenced by: +

+


substr_from:

+ + + + + + + FROM + + + + a_expr + + + + + +

referenced by: +

+


substr_for:

+ + + + + + + FOR + + + + a_expr + + + + + +

referenced by: +

+


window_definition:

+ + + + + + + + name + + + + AS + + + + window_specification + + + + + +

referenced by: +

+


index_hints_param_list:

+ + + + + + + + index_hints_param + + + + , + + + + +

referenced by: +

+


join_type:

+ + + + + + + FULL + + + LEFT + + + RIGHT + + + + join_outer + + + + INNER + + + + +

referenced by: +

+


join_qual:

+ + + + + + + USING + + + ( + + + + name_list + + + + ) + + + ON + + + + a_expr + + + + + +

referenced by: +

+


opt_name_parens:

+ + + + + + + ( + + + + name + + + + ) + + + + +

referenced by: +

+


index_hints_param:

+ + + + + + + FORCE_INDEX + + + = + + + + unrestricted_name + + + + NO_INDEX_JOIN + + + + +

referenced by: +

+


join_outer:

+ + + + + + + OUTER + + + + +

referenced by: +

+


generated by Railroad Diagram Generator

diff --git a/_includes/v20.2/sql/diagrams/grant.html b/_includes/v20.2/sql/diagrams/grant.html new file mode 100644 index 00000000000..4271082a93c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/grant.html @@ -0,0 +1,74 @@ +
+ + + + + + GRANT + + + ALL + + + CREATE + + + DROP + + + GRANT + + + SELECT + + + INSERT + + + DELETE + + + UPDATE + + + , + + + ON + + + TABLE + + + + table_name + + + + , + + + DATABASE + + + + database_name + + + + , + + + TO + + + + user_name + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/grant_privileges.html b/_includes/v20.2/sql/diagrams/grant_privileges.html new file mode 100644 index 00000000000..da7f44e5160 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/grant_privileges.html @@ -0,0 +1,74 @@ +
+ + + + + + GRANT + + + ALL + + + CREATE + + + GRANT + + + SELECT + + + DROP + + + INSERT + + + DELETE + + + UPDATE + + + , + + + ON + + + TABLE + + + + table_name + + + + , + + + DATABASE + + + + database_name + + + + , + + + TO + + + + user_name + + + + , + + + +
diff --git a/_includes/v20.2/sql/diagrams/grant_roles.html b/_includes/v20.2/sql/diagrams/grant_roles.html new file mode 100644 index 00000000000..f8eee0dc766 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/grant_roles.html @@ -0,0 +1,34 @@ +
+ + + + +GRANT + + +role_name + + +, + + +TO + + +user_name + + +, + + +WITH + + +ADMIN + + +OPTION + + + +
diff --git a/_includes/v20.2/sql/diagrams/import.html b/_includes/v20.2/sql/diagrams/import.html new file mode 100644 index 00000000000..4528fe2a3e2 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/import.html @@ -0,0 +1,72 @@ +
+ + + + + + IMPORT + + + TABLE + + + + table_name + + + + CREATE + + + USING + + + + create_table_file + + + + ( + + + + table_elem_list + + + + ) + + + CSV + + + DATA + + + ( + + + + file_to_import + + + + , + + + ) + + + WITH + + + + kv_option + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/import_csv.html b/_includes/v20.2/sql/diagrams/import_csv.html new file mode 100644 index 00000000000..20d4607a90b --- /dev/null +++ b/_includes/v20.2/sql/diagrams/import_csv.html @@ -0,0 +1,55 @@ +
+ + + + +IMPORT + + +TABLE + + +table_name + +CREATE + + +USING + + +file_location + +( + + +table_elem_list + +) + + +CSV + + +AVRO + + +DATA + + +( + + +file_location + +, + + +) + + +WITH + + +kv_option_list + +
diff --git a/_includes/v20.2/sql/diagrams/import_dump.html b/_includes/v20.2/sql/diagrams/import_dump.html new file mode 100644 index 00000000000..1c94207f03e --- /dev/null +++ b/_includes/v20.2/sql/diagrams/import_dump.html @@ -0,0 +1,27 @@ +
+ + + + +IMPORT + + +TABLE + + +table_name + +FROM + + +import_format + + +file_location + +WITH + + +kv_option_list + +
diff --git a/_includes/v20.2/sql/diagrams/import_into.html b/_includes/v20.2/sql/diagrams/import_into.html new file mode 100644 index 00000000000..0e8a2214d89 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/import_into.html @@ -0,0 +1,58 @@ +
+ + + + +IMPORT + + +INTO + + +table_name + +( + + +column_name + +, + + +) + + +CSV + + +AVRO + + +DATA + + +( + + +file_location + +, + + +) + + +WITH + + +option + += + + +value + +, + + +
diff --git a/_includes/v20.2/sql/diagrams/import_table.html b/_includes/v20.2/sql/diagrams/import_table.html new file mode 100644 index 00000000000..5dacfda2495 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/import_table.html @@ -0,0 +1,108 @@ +
+ + + + +IMPORT + + +import_format + + +( + + +file_location + + +) + + +file_location + + +TABLE + + + +table_name + + + +FROM + + +import_format + + +( + + +file_location + + +) + + +file_location + + +CREATE + + +USING + + +file_location + + +( + + + +table_elem_list + + + +) + + +import_format + + +DATA + + +( + + +file_location_list + + +) + + +WITH + + + +kv_option_list + + + +OPTIONS + + +( + + + +kv_option_list + + + +) + + + +
diff --git a/_includes/v20.2/sql/diagrams/index_def.html b/_includes/v20.2/sql/diagrams/index_def.html new file mode 100644 index 00000000000..52e04841c00 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/index_def.html @@ -0,0 +1,102 @@ +
+ + + + +UNIQUE + + +INDEX + + + +opt_index_name + + + +( + + + +index_elem + + + +, + + +) + + +USING + + +HASH + + +WITH + + +BUCKET_COUNT + + += + + +n_buckets + + +COVERING + + +STORING + + +( + + + +name_list + + + +) + + + +opt_interleave + + + + +opt_partition_by + + + +INVERTED + + +INDEX + + + +name + + + +( + + + +index_elem + + + +, + + +) + + + +
diff --git a/_includes/v20.2/sql/diagrams/insert.html b/_includes/v20.2/sql/diagrams/insert.html new file mode 100644 index 00000000000..c61450e7913 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/insert.html @@ -0,0 +1,62 @@ +
+ + + + +WITH + + +RECURSIVE + + +common_table_expr + +, + + +INSERT + + +INTO + + +table_name + +AS + + +table_alias_name + +( + + +column_name + +, + + +) + + +select_stmt + +DEFAULT + + +VALUES + + +on_conflict + +RETURNING + + +target_elem + +, + + +NOTHING + + +
diff --git a/_includes/v20.2/sql/diagrams/interleave.html b/_includes/v20.2/sql/diagrams/interleave.html new file mode 100644 index 00000000000..09bb9c35b5b --- /dev/null +++ b/_includes/v20.2/sql/diagrams/interleave.html @@ -0,0 +1,69 @@ +
+ + + + + + CREATE + + + TABLE + + + IF + + + NOT + + + EXISTS + + + + table_name + + + + ( + + + + table_definition + + + + ) + + + INTERLEAVE + + + IN + + + PARENT + + + + table_name + + + + ( + + + + name_list + + + + ) + + + + opt_partition_by + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/iso_level.html b/_includes/v20.2/sql/diagrams/iso_level.html new file mode 100644 index 00000000000..c14993cb0cf --- /dev/null +++ b/_includes/v20.2/sql/diagrams/iso_level.html @@ -0,0 +1,29 @@ +
+ + + + + + READ + + + UNCOMMITTED + + + COMMITTED + + + SNAPSHOT + + + REPEATABLE + + + READ + + + SERIALIZABLE + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/joined_table.html b/_includes/v20.2/sql/diagrams/joined_table.html new file mode 100644 index 00000000000..68b66314702 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/joined_table.html @@ -0,0 +1,100 @@ +
+ + + + +( + + + +joined_table + + + +) + + + +table_ref + + + +CROSS + + +NATURAL + + +FULL + + +LEFT + + +RIGHT + + +OUTER + + +INNER + + +JOIN + + + +table_ref + + + +FULL + + +LEFT + + +RIGHT + + +OUTER + + +INNER + + +JOIN + + + +table_ref + + + +USING + + +( + + + +name + + + +, + + +) + + +ON + + + +a_expr + + + + +
diff --git a/_includes/v20.2/sql/diagrams/limit_clause.html b/_includes/v20.2/sql/diagrams/limit_clause.html new file mode 100644 index 00000000000..98d5114a88e --- /dev/null +++ b/_includes/v20.2/sql/diagrams/limit_clause.html @@ -0,0 +1,38 @@ +
+ + + + +LIMIT + + + +count + + + +FETCH + + +FIRST + + +NEXT + + + +count + + + +ROW + + +ROWS + + +ONLY + + + +
diff --git a/_includes/v20.2/sql/diagrams/not_null_column_level.html b/_includes/v20.2/sql/diagrams/not_null_column_level.html new file mode 100644 index 00000000000..52e17e9d57d --- /dev/null +++ b/_includes/v20.2/sql/diagrams/not_null_column_level.html @@ -0,0 +1,59 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + NOT NULL + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/offset_clause.html b/_includes/v20.2/sql/diagrams/offset_clause.html new file mode 100644 index 00000000000..d6dc4873ee5 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/offset_clause.html @@ -0,0 +1,26 @@ +
+ + + + +OFFSET + + + +a_expr + + + + +c_expr + + + +ROW + + +ROWS + + + +
diff --git a/_includes/v20.2/sql/diagrams/on_conflict.html b/_includes/v20.2/sql/diagrams/on_conflict.html new file mode 100644 index 00000000000..1edb951f1b9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/on_conflict.html @@ -0,0 +1,78 @@ +
+ + + + +ON + + +CONFLICT + + +( + + +name + +, + + +) + + +DO + + +UPDATE + + +SET + + +column_name + += + + +a_expr + +( + + +column_name + +, + + +) + + += + + +( + + +select_stmt + + +a_expr + +, + + +a_expr + +, + + +) + + +, + + +NOTHING + + +
diff --git a/_includes/v20.2/sql/diagrams/opt_frame_clause.html b/_includes/v20.2/sql/diagrams/opt_frame_clause.html new file mode 100644 index 00000000000..17ebda5d4ef --- /dev/null +++ b/_includes/v20.2/sql/diagrams/opt_frame_clause.html @@ -0,0 +1,28 @@ +
+ + + + +RANGE + + +ROWS + + +GROUPS + + +BETWEEN + + +frame_bound + +AND + + +frame_bound + + +opt_frame_exclusion + +
diff --git a/_includes/v20.2/sql/diagrams/opt_interleave.html b/_includes/v20.2/sql/diagrams/opt_interleave.html new file mode 100644 index 00000000000..5825c01b310 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/opt_interleave.html @@ -0,0 +1,33 @@ +
+ + + + + + INTERLEAVE + + + IN + + + PARENT + + + + table_name + + + + ( + + + + name_list + + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/opt_temp_create_table.html b/_includes/v20.2/sql/diagrams/opt_temp_create_table.html new file mode 100644 index 00000000000..0eadd175eb2 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/opt_temp_create_table.html @@ -0,0 +1,19 @@ +
+ + + + +LOCAL + + +GLOBAL + + +TEMPORARY + + +TEMP + + + +
diff --git a/_includes/v20.2/sql/diagrams/pause_job.html b/_includes/v20.2/sql/diagrams/pause_job.html new file mode 100644 index 00000000000..3d0949c6088 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/pause_job.html @@ -0,0 +1,24 @@ +
+ + + + +PAUSE + + +JOB + + +job_id + + +JOBS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/primary_key_column_level.html b/_includes/v20.2/sql/diagrams/primary_key_column_level.html new file mode 100644 index 00000000000..f938b641654 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/primary_key_column_level.html @@ -0,0 +1,59 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + PRIMARY KEY + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/primary_key_columns_level.html b/_includes/v20.2/sql/diagrams/primary_key_columns_level.html new file mode 100644 index 00000000000..9f9c6633527 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/primary_key_columns_level.html @@ -0,0 +1,54 @@ +
+ + + + + + CREATE + + + TABLE + + + + any_name + + + + ( + + + + name + + + + + typename + + + + PRIMARY KEY + + + , + + + + column_def + + + + + other_constraints + + + + ) + + + ) + + + +
diff --git a/_includes/v20.2/sql/diagrams/primary_key_table_level.html b/_includes/v20.2/sql/diagrams/primary_key_table_level.html new file mode 100644 index 00000000000..db8ece49c39 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/primary_key_table_level.html @@ -0,0 +1,63 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_def + + + + , + + + CONSTRAINT + + + + name + + + + PRIMARY KEY + + + ( + + + + column_name + + + + , + + + ) + + + + table_constraints + + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/release_savepoint.html b/_includes/v20.2/sql/diagrams/release_savepoint.html new file mode 100644 index 00000000000..194ce6573ca --- /dev/null +++ b/_includes/v20.2/sql/diagrams/release_savepoint.html @@ -0,0 +1,19 @@ +
+ + + + + + RELEASE + + + SAVEPOINT + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/rename_column.html b/_includes/v20.2/sql/diagrams/rename_column.html new file mode 100644 index 00000000000..2d275bc9de7 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_column.html @@ -0,0 +1,44 @@ +
+ + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + table_name + + + + RENAME + + + COLUMN + + + + current_name + + + + TO + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/rename_constraint.html b/_includes/v20.2/sql/diagrams/rename_constraint.html new file mode 100644 index 00000000000..36b2c9dfe1f --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_constraint.html @@ -0,0 +1,33 @@ +
+ + + + +ALTER + + +TABLE + + +IF + + +EXISTS + + +table_name + +RENAME + + +CONSTRAINT + + +current_name + +TO + + +name + +
diff --git a/_includes/v20.2/sql/diagrams/rename_database.html b/_includes/v20.2/sql/diagrams/rename_database.html new file mode 100644 index 00000000000..ce9ddd3ddba --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_database.html @@ -0,0 +1,30 @@ +
+ + + + + + ALTER + + + DATABASE + + + + name + + + + RENAME + + + TO + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/rename_index.html b/_includes/v20.2/sql/diagrams/rename_index.html new file mode 100644 index 00000000000..82ed2e90255 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_index.html @@ -0,0 +1,33 @@ +
+ + + + +ALTER + + +INDEX + + +IF + + +EXISTS + + +table_name + +@ + + +index_name + +RENAME + + +TO + + +index_name + +
diff --git a/_includes/v20.2/sql/diagrams/rename_sequence.html b/_includes/v20.2/sql/diagrams/rename_sequence.html new file mode 100644 index 00000000000..a564d9db425 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_sequence.html @@ -0,0 +1,36 @@ +
+ + + + + + ALTER + + + SEQUENCE + + + IF + + + EXISTS + + + + current_name + + + + RENAME + + + TO + + + + new_name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/rename_table.html b/_includes/v20.2/sql/diagrams/rename_table.html new file mode 100644 index 00000000000..316c56482eb --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rename_table.html @@ -0,0 +1,36 @@ +
+ + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + current_name + + + + RENAME + + + TO + + + + new_name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/reset_csetting.html b/_includes/v20.2/sql/diagrams/reset_csetting.html new file mode 100644 index 00000000000..49e120ffc69 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/reset_csetting.html @@ -0,0 +1,22 @@ +
+ + + + + + RESET + + + CLUSTER + + + SETTING + + + + var_name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/reset_session.html b/_includes/v20.2/sql/diagrams/reset_session.html new file mode 100644 index 00000000000..0a47ec52d49 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/reset_session.html @@ -0,0 +1,19 @@ +
+ + + + + + RESET + + + SESSION + + + + session_var + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/restore.html b/_includes/v20.2/sql/diagrams/restore.html new file mode 100644 index 00000000000..c3fadf73e98 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/restore.html @@ -0,0 +1,55 @@ +
+ + + + +RESTORE + + +TABLE + + +table_pattern + +, + + +DATABASE + + +database_name + +, + + +FROM + + +full_backup_location + + +incremental_backup_location + +, + + +AS + + +OF + + +SYSTEM + + +TIME + + +timestamp + +WITH + + +kv_option_list + +
diff --git a/_includes/v20.2/sql/diagrams/resume_job.html b/_includes/v20.2/sql/diagrams/resume_job.html new file mode 100644 index 00000000000..552bef86bce --- /dev/null +++ b/_includes/v20.2/sql/diagrams/resume_job.html @@ -0,0 +1,24 @@ +
+ + + + +RESUME + + +JOB + + +job_id + + +JOBS + + + +select_stmt + + + + +
diff --git a/_includes/v20.2/sql/diagrams/revoke.html b/_includes/v20.2/sql/diagrams/revoke.html new file mode 100644 index 00000000000..f321d3a87a8 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/revoke.html @@ -0,0 +1,74 @@ +
+ + + + + + REVOKE + + + ALL + + + CREATE + + + DROP + + + GRANT + + + SELECT + + + INSERT + + + DELETE + + + UPDATE + + + , + + + ON + + + TABLE + + + + table_name + + + + , + + + DATABASE + + + + database_name + + + + , + + + FROM + + + + user_name + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/revoke_privileges.html b/_includes/v20.2/sql/diagrams/revoke_privileges.html new file mode 100644 index 00000000000..a6f9a1dee8e --- /dev/null +++ b/_includes/v20.2/sql/diagrams/revoke_privileges.html @@ -0,0 +1,74 @@ +
+ + + + + + REVOKE + + + ALL + + + CREATE + + + GRANT + + + SELECT + + + DROP + + + INSERT + + + DELETE + + + UPDATE + + + , + + + ON + + + TABLE + + + + table_name + + + + , + + + DATABASE + + + + database_name + + + + , + + + FROM + + + + user_name + + + + , + + + +
diff --git a/_includes/v20.2/sql/diagrams/revoke_roles.html b/_includes/v20.2/sql/diagrams/revoke_roles.html new file mode 100644 index 00000000000..a30aee75474 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/revoke_roles.html @@ -0,0 +1,34 @@ +
+ + + + +REVOKE + + +ADMIN + + +OPTION + + +FOR + + +role_name + + +, + + +FROM + + +user_name + + +, + + + +
diff --git a/_includes/v20.2/sql/diagrams/rollback_transaction.html b/_includes/v20.2/sql/diagrams/rollback_transaction.html new file mode 100644 index 00000000000..e981a160929 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/rollback_transaction.html @@ -0,0 +1,17 @@ +
+ + + + +ROLLBACK + + +TO + + +SAVEPOINT + + +savepoint_name + +
diff --git a/_includes/v20.2/sql/diagrams/savepoint.html b/_includes/v20.2/sql/diagrams/savepoint.html new file mode 100644 index 00000000000..9b7dc70608b --- /dev/null +++ b/_includes/v20.2/sql/diagrams/savepoint.html @@ -0,0 +1,16 @@ +
+ + + + + + SAVEPOINT + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/select.html b/_includes/v20.2/sql/diagrams/select.html new file mode 100644 index 00000000000..9fc522fe5c9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/select.html @@ -0,0 +1,116 @@ + diff --git a/_includes/v20.2/sql/diagrams/select_clause.html b/_includes/v20.2/sql/diagrams/select_clause.html new file mode 100644 index 00000000000..535480ba3ba --- /dev/null +++ b/_includes/v20.2/sql/diagrams/select_clause.html @@ -0,0 +1,53 @@ + diff --git a/_includes/v20.2/sql/diagrams/set_application_name.html b/_includes/v20.2/sql/diagrams/set_application_name.html new file mode 100644 index 00000000000..cb84775aa12 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_application_name.html @@ -0,0 +1,25 @@ +
+ + + + + + SET + + + APPLICATION_NAME + + + TO + + + = + + + + application_name + + + + +
diff --git a/_includes/v20.2/sql/diagrams/set_cluster_setting.html b/_includes/v20.2/sql/diagrams/set_cluster_setting.html new file mode 100644 index 00000000000..b6554c7be52 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_cluster_setting.html @@ -0,0 +1,36 @@ +
+ + + + + + SET + + + CLUSTER + + + SETTING + + + + var_name + + + + = + + + TO + + + + var_value + + + + DEFAULT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/set_database.html b/_includes/v20.2/sql/diagrams/set_database.html new file mode 100644 index 00000000000..c8b1ddb887f --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_database.html @@ -0,0 +1,25 @@ +
+ + + + + + SET + + + DATABASE + + + TO + + + = + + + + database_name + + + + +
diff --git a/_includes/v20.2/sql/diagrams/set_operation.html b/_includes/v20.2/sql/diagrams/set_operation.html new file mode 100644 index 00000000000..aa0e63023dc --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_operation.html @@ -0,0 +1,32 @@ +
+ + + + + +select_clause + + + +UNION + + +INTERSECT + + +EXCEPT + + +ALL + + +DISTINCT + + + +select_clause + + + + +
diff --git a/_includes/v20.2/sql/diagrams/set_transaction.html b/_includes/v20.2/sql/diagrams/set_transaction.html new file mode 100644 index 00000000000..3b3ca38af19 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_transaction.html @@ -0,0 +1,50 @@ +
+ + + + +SET + + +TRANSACTION + + +PRIORITY + + +LOW + + +NORMAL + + +HIGH + + +READ + + +ONLY + + +WRITE + + +AS + + +OF + + +SYSTEM + + +TIME + + +a_expr + +, + + +
diff --git a/_includes/v20.2/sql/diagrams/set_var.html b/_includes/v20.2/sql/diagrams/set_var.html new file mode 100644 index 00000000000..96bb04e7cf6 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/set_var.html @@ -0,0 +1,33 @@ +
+ + + + + + SET + + + SESSION + + + + var_name + + + + TO + + + = + + + + var_value + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_backup.html b/_includes/v20.2/sql/diagrams/show_backup.html new file mode 100644 index 00000000000..27b69b5ab8a --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_backup.html @@ -0,0 +1,22 @@ +
+ + + + +SHOW + + +BACKUP + + +SCHEMAS + + +location + +WITH + + +kv_option_list + +
diff --git a/_includes/v20.2/sql/diagrams/show_cluster_setting.html b/_includes/v20.2/sql/diagrams/show_cluster_setting.html new file mode 100644 index 00000000000..7aeef4c1ad3 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_cluster_setting.html @@ -0,0 +1,35 @@ +
+ + + + +SHOW + + +CLUSTER + + +SETTING + + +var_name + +ALL + + +SETTINGS + + +ALL + + +PUBLIC + + +CLUSTER + + +SETTINGS + + +
diff --git a/_includes/v20.2/sql/diagrams/show_columns.html b/_includes/v20.2/sql/diagrams/show_columns.html new file mode 100644 index 00000000000..9da18b6612a --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_columns.html @@ -0,0 +1,23 @@ +
+ + + + +SHOW + + +COLUMNS + + +FROM + + +table_name + +WITH + + +COMMENT + + +
diff --git a/_includes/v20.2/sql/diagrams/show_constraints.html b/_includes/v20.2/sql/diagrams/show_constraints.html new file mode 100644 index 00000000000..9c520ae9bc6 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_constraints.html @@ -0,0 +1,25 @@ +
+ + + + + + SHOW + + + CONSTRAINT + + + CONSTRAINTS + + + FROM + + + + table_name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_create.html b/_includes/v20.2/sql/diagrams/show_create.html new file mode 100644 index 00000000000..09c0fa4c2a1 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_create.html @@ -0,0 +1,16 @@ +
+ + + + +SHOW + + +CREATE + + +object_name + + + +
diff --git a/_includes/v20.2/sql/diagrams/show_database.html b/_includes/v20.2/sql/diagrams/show_database.html new file mode 100644 index 00000000000..dbae975fd37 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_database.html @@ -0,0 +1,14 @@ +
+ + + + + + SHOW + + + DATABASE + + + +
diff --git a/_includes/v20.2/sql/diagrams/show_databases.html b/_includes/v20.2/sql/diagrams/show_databases.html new file mode 100644 index 00000000000..0270318301c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_databases.html @@ -0,0 +1,18 @@ +
+ + + + +SHOW + + +DATABASES + + +WITH + + +COMMENT + + +
diff --git a/_includes/v20.2/sql/diagrams/show_grants.html b/_includes/v20.2/sql/diagrams/show_grants.html new file mode 100644 index 00000000000..92a7932dc22 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_grants.html @@ -0,0 +1,61 @@ +
+ + + + + + SHOW + + + GRANTS + + + ON + + + ROLE + + + + role_name + + + + , + + + TABLE + + + + table_name + + + + , + + + DATABASE + + + + database_name + + + + , + + + FOR + + + + user_name + + + + , + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_indexes.html b/_includes/v20.2/sql/diagrams/show_indexes.html new file mode 100644 index 00000000000..f640cb0b1ee --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_indexes.html @@ -0,0 +1,34 @@ +
+ + + + +SHOW + + +INDEX + + +INDEXES + + +KEYS + + +FROM + + +table_name + +DATABASE + + +database_name + +WITH + + +COMMENT + + +
diff --git a/_includes/v20.2/sql/diagrams/show_jobs.html b/_includes/v20.2/sql/diagrams/show_jobs.html new file mode 100644 index 00000000000..6fdd6553de2 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_jobs.html @@ -0,0 +1,37 @@ +
+ + + + +SHOW + + +AUTOMATIC + + +JOBS + + +JOBS + + +WHEN + + +COMPLETE + + +select_stmt + +JOB + + +WHEN + + +COMPLETE + + +job_id + +
diff --git a/_includes/v20.2/sql/diagrams/show_keys.html b/_includes/v20.2/sql/diagrams/show_keys.html new file mode 100644 index 00000000000..26a6dabc087 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_keys.html @@ -0,0 +1,8 @@ +
+ + + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_locality.html b/_includes/v20.2/sql/diagrams/show_locality.html new file mode 100644 index 00000000000..2ca711fbdad --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_locality.html @@ -0,0 +1,12 @@ +
+ + + + +SHOW + + +LOCALITY + + +
diff --git a/_includes/v20.2/sql/diagrams/show_partitions.html b/_includes/v20.2/sql/diagrams/show_partitions.html new file mode 100644 index 00000000000..aeb6156fbd3 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_partitions.html @@ -0,0 +1,30 @@ +
+ + + + +SHOW + + +PARTITIONS + + +FROM + + +TABLE + + +table_name + +DATABASE + + +database_name + +INDEX + + +table_index_name + +
diff --git a/_includes/v20.2/sql/diagrams/show_queries.html b/_includes/v20.2/sql/diagrams/show_queries.html new file mode 100644 index 00000000000..26376243dac --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_queries.html @@ -0,0 +1,20 @@ +
+ + + + + + SHOW + + + CLUSTER + + + LOCAL + + + QUERIES + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_range_for_row.html b/_includes/v20.2/sql/diagrams/show_range_for_row.html new file mode 100644 index 00000000000..ab882008526 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_range_for_row.html @@ -0,0 +1,42 @@ +
+ + + + +SHOW + + +RANGE + + +FROM + + +TABLE + + +table_name + +INDEX + + +table_index_name + +FOR + + +ROW + + +( + + +row_vals + +, + + +) + + +
diff --git a/_includes/v20.2/sql/diagrams/show_ranges.html b/_includes/v20.2/sql/diagrams/show_ranges.html new file mode 100644 index 00000000000..bd1553dfe05 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_ranges.html @@ -0,0 +1,30 @@ +
+ + + + +SHOW + + +RANGES + + +FROM + + +TABLE + + +table_name + +INDEX + + +table_index_name + +DATABASE + + +database_name + +
diff --git a/_includes/v20.2/sql/diagrams/show_roles.html b/_includes/v20.2/sql/diagrams/show_roles.html new file mode 100644 index 00000000000..fd508395e0b --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_roles.html @@ -0,0 +1,14 @@ +
+ + + + + + SHOW + + + ROLES + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_savepoint_status.html b/_includes/v20.2/sql/diagrams/show_savepoint_status.html new file mode 100644 index 00000000000..7fc1c8fa52d --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_savepoint_status.html @@ -0,0 +1,15 @@ +
+ + + + +SHOW + + +SAVEPOINT + + +STATUS + + +
diff --git a/_includes/v20.2/sql/diagrams/show_schemas.html b/_includes/v20.2/sql/diagrams/show_schemas.html new file mode 100644 index 00000000000..efa07764533 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_schemas.html @@ -0,0 +1,22 @@ +
+ + + + + + SHOW + + + SCHEMAS + + + FROM + + + + name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_sequences.html b/_includes/v20.2/sql/diagrams/show_sequences.html new file mode 100644 index 00000000000..4f3fe915c12 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_sequences.html @@ -0,0 +1,17 @@ +
+ + + + +SHOW + + +SEQUENCES + + +FROM + + +name + +
diff --git a/_includes/v20.2/sql/diagrams/show_sessions.html b/_includes/v20.2/sql/diagrams/show_sessions.html new file mode 100644 index 00000000000..3b2aa5b16ee --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_sessions.html @@ -0,0 +1,20 @@ +
+ + + + + + SHOW + + + CLUSTER + + + LOCAL + + + SESSIONS + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_stats.html b/_includes/v20.2/sql/diagrams/show_stats.html new file mode 100644 index 00000000000..0e350b93c0f --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_stats.html @@ -0,0 +1,20 @@ +
+ + + + +SHOW + + +STATISTICS + + +FOR + + +TABLE + + +table_name + +
diff --git a/_includes/v20.2/sql/diagrams/show_tables.html b/_includes/v20.2/sql/diagrams/show_tables.html new file mode 100644 index 00000000000..84b221efaf2 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_tables.html @@ -0,0 +1,28 @@ +
+ + + + +SHOW + + +TABLES + + +FROM + + +database_name + +. + + +schema_name + +WITH + + +COMMENT + + +
diff --git a/_includes/v20.2/sql/diagrams/show_timezone.html b/_includes/v20.2/sql/diagrams/show_timezone.html new file mode 100644 index 00000000000..ad655fe7028 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_timezone.html @@ -0,0 +1,17 @@ +
+ + + + + + SHOW + + + TIME + + + ZONE + + + +
diff --git a/_includes/v20.2/sql/diagrams/show_trace.html b/_includes/v20.2/sql/diagrams/show_trace.html new file mode 100644 index 00000000000..37271dc87b5 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_trace.html @@ -0,0 +1,25 @@ +
+ + + + +SHOW + + +COMPACT + + +KV + + +TRACE + + +FOR + + +SESSION + + + +
diff --git a/_includes/v20.2/sql/diagrams/show_transaction.html b/_includes/v20.2/sql/diagrams/show_transaction.html new file mode 100644 index 00000000000..26a6dabc087 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_transaction.html @@ -0,0 +1,8 @@ +
+ + + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_users.html b/_includes/v20.2/sql/diagrams/show_users.html new file mode 100644 index 00000000000..7c33b7f00b4 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_users.html @@ -0,0 +1,14 @@ +
+ + + + + + SHOW + + + USERS + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_var.html b/_includes/v20.2/sql/diagrams/show_var.html new file mode 100644 index 00000000000..fb7ec6f4ce8 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_var.html @@ -0,0 +1,20 @@ +
+ + + + + + SHOW + + + SESSION + + + var_name + + + ALL + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/show_zone.html b/_includes/v20.2/sql/diagrams/show_zone.html new file mode 100644 index 00000000000..83052dd1d5c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/show_zone.html @@ -0,0 +1,73 @@ +
+ + + + +SHOW + + +ZONE + + +CONFIGURATION + + +FOR + + +RANGE + + +zone_name + +DATABASE + + +database_name + +TABLE + + +table_name + +PARTITION + + +partition_name + +PARTITION + + +partition_name + +OF + + +TABLE + + +table_name + +INDEX + + +table_name + +@ + + +index_name + +CONFIGURATIONS + + +ALL + + +ZONE + + +CONFIGURATIONS + + +
diff --git a/_includes/v20.2/sql/diagrams/simple_select_clause.html b/_includes/v20.2/sql/diagrams/simple_select_clause.html new file mode 100644 index 00000000000..fd45ea37754 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/simple_select_clause.html @@ -0,0 +1,107 @@ +
+ + + + +SELECT + + +ALL + + +DISTINCT + + +ON + + +( + + + +a_expr + + + +, + + +) + + + +target_elem + + + +, + + +FROM + + + +table_ref + + + +, + + +AS + + +OF + + +SYSTEM + + +TIME + + + +a_expr + + + +WHERE + + + +a_expr + + + +GROUP + + +BY + + + +a_expr + + + +, + + +HAVING + + + +a_expr + + + +WINDOW + + + +window_definition_list + + + + +
diff --git a/_includes/v20.2/sql/diagrams/sort_clause.html b/_includes/v20.2/sql/diagrams/sort_clause.html new file mode 100644 index 00000000000..59aababe951 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/sort_clause.html @@ -0,0 +1,69 @@ +
+ + + + +ORDER + + +BY + + + +a_expr + + + +ASC + + +DESC + + +NULLS + + +FIRST + + +LAST + + +PRIMARY + + +KEY + + + +table_name + + + +INDEX + + + +table_name + + + +@ + + + +index_name + + + +ASC + + +DESC + + +, + + + +
diff --git a/_includes/v20.2/sql/diagrams/split_index_at.html b/_includes/v20.2/sql/diagrams/split_index_at.html new file mode 100644 index 00000000000..51daee7e3c7 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/split_index_at.html @@ -0,0 +1,35 @@ +
+ + + + +ALTER + + +INDEX + + +table_name + +@ + + +index_name + +SPLIT + + +AT + + +select_stmt + +WITH + + +EXPIRATION + + +a_expr + +
diff --git a/_includes/v20.2/sql/diagrams/split_table_at.html b/_includes/v20.2/sql/diagrams/split_table_at.html new file mode 100644 index 00000000000..2b7b43c5a59 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/split_table_at.html @@ -0,0 +1,30 @@ +
+ + + + +ALTER + + +TABLE + + +table_name + +SPLIT + + +AT + + +select_stmt + +WITH + + +EXPIRATION + + +a_expr + +
diff --git a/_includes/v20.2/sql/diagrams/stmt_block.html b/_includes/v20.2/sql/diagrams/stmt_block.html new file mode 100644 index 00000000000..3d9d766edab --- /dev/null +++ b/_includes/v20.2/sql/diagrams/stmt_block.html @@ -0,0 +1,16891 @@ +
+ + +

stmt_block:

+ + + + + + + + stmt + + + + + +

no references


stmt:

+ + + + + + + HELPTOKEN + + + + preparable_stmt + + + + + copy_from_stmt + + + + + comment_stmt + + + + + execute_stmt + + + + + deallocate_stmt + + + + + discard_stmt + + + + + grant_stmt + + + + + prepare_stmt + + + + + revoke_stmt + + + + + savepoint_stmt + + + + + release_stmt + + + + + nonpreparable_set_stmt + + + + + transaction_stmt + + + + + +

referenced by: +

+


preparable_stmt:

+ + + + + + + + alter_stmt + + + + + backup_stmt + + + + + cancel_stmt + + + + + create_stmt + + + + + delete_stmt + + + + + drop_stmt + + + + + explain_stmt + + + + + import_stmt + + + + + insert_stmt + + + + + pause_stmt + + + + + reset_stmt + + + + + restore_stmt + + + + + resume_stmt + + + + + export_stmt + + + + + scrub_stmt + + + + + select_stmt + + + + + preparable_set_stmt + + + + + show_stmt + + + + + truncate_stmt + + + + + update_stmt + + + + + upsert_stmt + + + + + +

referenced by: +

+


copy_from_stmt:

+ + + + + + + COPY + + + + table_name + + + + + opt_column_list + + + + FROM + + + STDIN + + + + opt_with_options + + + + + +

referenced by: +

+


comment_stmt:

+ + + + + + + COMMENT + + + ON + + + DATABASE + + + + database_name + + + + TABLE + + + + table_name + + + + COLUMN + + + + column_path + + + + INDEX + + + + table_index_name + + + + IS + + + + comment_text + + + + + +

referenced by: +

+


execute_stmt:

+ + + + + + + EXECUTE + + + + table_alias_name + + + + + execute_param_clause + + + + + +

referenced by: +

+


deallocate_stmt:

+ + + + + + + DEALLOCATE + + + PREPARE + + + + name + + + + ALL + + + + +

referenced by: +

+


discard_stmt:

+ + + + + + + DISCARD + + + ALL + + + + +

referenced by: +

+


grant_stmt:

+ + + + + + + GRANT + + + + privileges + + + + ON + + + + targets + + + + TO + + + + name_list + + + + + privilege_list + + + + TO + + + + name_list + + + + WITH + + + ADMIN + + + OPTION + + + + +

referenced by: +

+


prepare_stmt:

+ + + + + + + PREPARE + + + + table_alias_name + + + + + prep_type_clause + + + + AS + + + + preparable_stmt + + + + + +

referenced by: +

+


revoke_stmt:

+ + + + + + + REVOKE + + + + privileges + + + + ON + + + + targets + + + + ADMIN + + + OPTION + + + FOR + + + + privilege_list + + + + FROM + + + + name_list + + + + + +

referenced by: +

+


savepoint_stmt:

+ + + + + + + SAVEPOINT + + + + name + + + + + +

referenced by: +

+


release_stmt:

+ + + + + + + RELEASE + + + + savepoint_name + + + + + +

referenced by: +

+


nonpreparable_set_stmt:

+ + + + + + + + set_transaction_stmt + + + + + +

referenced by: +

+


transaction_stmt:

+ + + + + + + + begin_stmt + + + + + commit_stmt + + + + + rollback_stmt + + + + + abort_stmt + + + + + +

referenced by: +

+


alter_stmt:

+ + + + + + + + alter_ddl_stmt + + + + + alter_role_stmt + + + + + +

referenced by: +

+


backup_stmt:

+ + + + + + + BACKUP + + + + targets + + + + TO + + + + partitioned_backup + + + + + opt_as_of_clause + + + + + opt_incremental + + + + + opt_with_options + + + + + +

referenced by: +

+


cancel_stmt:

+ + + + + + + + cancel_jobs_stmt + + + + + cancel_queries_stmt + + + + + cancel_sessions_stmt + + + + + +

referenced by: +

+


create_stmt:

+ + + + + + + + create_role_stmt + + + + + create_ddl_stmt + + + + + create_stats_stmt + + + + + +

referenced by: +

+


delete_stmt:

+ + + + + + + + opt_with_clause + + + + DELETE + + + FROM + + + + table_expr_opt_alias_idx + + + + + opt_where_clause + + + + + opt_sort_clause + + + + + opt_limit_clause + + + + + returning_clause + + + + + +

referenced by: +

+


drop_stmt:

+ + + + + + + + drop_ddl_stmt + + + + + drop_role_stmt + + + + + +

referenced by: +

+


explain_stmt:

+ + + + + + + EXPLAIN + + + ANALYZE + + + ANALYSE + + + ( + + + + explain_option_list + + + + ) + + + + preparable_stmt + + + + + +

referenced by: +

+


import_stmt:

+ + + + + + + IMPORT + + + + import_format + + + + + string_or_placeholder + + + + TABLE + + + + table_name + + + + FROM + + + + import_format + + + + + string_or_placeholder + + + + CREATE + + + USING + + + + string_or_placeholder + + + + ( + + + + table_elem_list + + + + ) + + + + import_format + + + + DATA + + + ( + + + + string_or_placeholder_list + + + + ) + + + INTO + + + + table_name + + + + ( + + + + insert_column_list + + + + ) + + + + import_format + + + + DATA + + + ( + + + + string_or_placeholder_list + + + + ) + + + + opt_with_options + + + + + +

referenced by: +

+


insert_stmt:

+ + + + + + + + opt_with_clause + + + + INSERT + + + INTO + + + + insert_target + + + + + insert_rest + + + + + on_conflict + + + + + returning_clause + + + + + +

referenced by: +

+


pause_stmt:

+ + + + + + + PAUSE + + + JOB + + + + a_expr + + + + JOBS + + + + select_stmt + + + + + +

referenced by: +

+


reset_stmt:

+ + + + + + + + reset_session_stmt + + + + + reset_csetting_stmt + + + + + +

referenced by: +

+


restore_stmt:

+ + + + + + + RESTORE + + + + targets + + + + FROM + + + + partitioned_backup_list + + + + + opt_as_of_clause + + + + + opt_with_options + + + + + +

referenced by: +

+


resume_stmt:

+ + + + + + + RESUME + + + JOB + + + + a_expr + + + + JOBS + + + + select_stmt + + + + + +

referenced by: +

+


export_stmt:

+ + + + + + + EXPORT + + + INTO + + + + import_format + + + + + string_or_placeholder + + + + + opt_with_options + + + + FROM + + + + select_stmt + + + + + +

referenced by: +

+


scrub_stmt:

+ + + + + + + + scrub_table_stmt + + + + + scrub_database_stmt + + + + + +

referenced by: +

+


select_stmt:

+ + + + + + + + select_no_parens + + + + + select_with_parens + + + + + +

referenced by: +

+


preparable_set_stmt:

+ + + + + + + + set_session_stmt + + + + + set_csetting_stmt + + + + + use_stmt + + + + + +

referenced by: +

+


show_stmt:

+ + + + + + + + show_backup_stmt + + + + + show_columns_stmt + + + + + show_constraints_stmt + + + + + show_create_stmt + + + + + show_csettings_stmt + + + + + show_databases_stmt + + + + + show_grants_stmt + + + + + show_indexes_stmt + + + + + show_partitions_stmt + + + + + show_jobs_stmt + + + + + show_queries_stmt + + + + + show_ranges_stmt + + + + + show_range_for_row_stmt + + + + + show_roles_stmt + + + + + show_savepoint_stmt + + + + + show_schemas_stmt + + + + + show_sequences_stmt + + + + + show_session_stmt + + + + + show_sessions_stmt + + + + + show_stats_stmt + + + + + show_tables_stmt + + + + + show_trace_stmt + + + + + show_users_stmt + + + + + show_zone_stmt + + + + + +

referenced by: +

+


truncate_stmt:

+ + + + + + + TRUNCATE + + + + opt_table + + + + + relation_expr_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


update_stmt:

+ + + + + + + + opt_with_clause + + + + UPDATE + + + + table_expr_opt_alias_idx + + + + SET + + + + set_clause_list + + + + + opt_from_list + + + + + opt_where_clause + + + + + opt_sort_clause + + + + + opt_limit_clause + + + + + returning_clause + + + + + +

referenced by: +

+


upsert_stmt:

+ + + + + + + + opt_with_clause + + + + UPSERT + + + INTO + + + + insert_target + + + + + insert_rest + + + + + returning_clause + + + + + +

referenced by: +

+


table_name:

+ + + + + + + + db_object_name + + + + + +

referenced by: +

+


opt_column_list:

+ + + + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_with_options:

+ + + + + + + WITH + + + + kv_option_list + + + + OPTIONS + + + ( + + + + kv_option_list + + + + ) + + + + +

referenced by: +

+


database_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


comment_text:

+ + + + + + + SCONST + + + NULL + + + + +

referenced by: +

+


column_path:

+ + + + + + + + name + + + + + prefixed_column_path + + + + + +

referenced by: +

+


table_index_name:

+ + + + + + + + table_name + + + + @ + + + + index_name + + + + + standalone_index_name + + + + + +

referenced by: +

+


table_alias_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


execute_param_clause:

+ + + + + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + +

referenced by: +

+


privileges:

+ + + + + + + ALL + + + + privilege_list + + + + + +

referenced by: +

+


targets:

+ + + + + + + identifier + + + + col_name_keyword + + + + + unreserved_keyword + + + + + complex_table_pattern + + + + + table_pattern + + + + , + + + TABLE + + + + table_pattern_list + + + + DATABASE + + + + name_list + + + + + +

referenced by: +

+


name_list:

+ + + + + + + + name + + + + , + + + + +

referenced by: +

+


privilege_list:

+ + + + + + + + privilege + + + + , + + + + +

referenced by: +

+


prep_type_clause:

+ + + + + + + ( + + + + type_list + + + + ) + + + + +

referenced by: +

+


savepoint_name:

+ + + + + + + SAVEPOINT + + + + name + + + + + +

referenced by: +

+


set_transaction_stmt:

+ + + + + + + SET + + + SESSION + + + TRANSACTION + + + + transaction_mode_list + + + + + +

referenced by: +

+


begin_stmt:

+ + + + + + + BEGIN + + + + opt_transaction + + + + START + + + TRANSACTION + + + + begin_transaction + + + + + +

referenced by: +

+


commit_stmt:

+ + + + + + + COMMIT + + + END + + + + opt_transaction + + + + + +

referenced by: +

+


rollback_stmt:

+ + + + + + + ROLLBACK + + + + opt_transaction + + + + TO + + + + savepoint_name + + + + + +

referenced by: +

+


abort_stmt:

+ + + + + + + ABORT + + + + opt_abort_mod + + + + + +

referenced by: +

+


alter_ddl_stmt:

+ + + + + + + + alter_table_stmt + + + + + alter_index_stmt + + + + + alter_view_stmt + + + + + alter_sequence_stmt + + + + + alter_database_stmt + + + + + alter_range_stmt + + + + + alter_partition_stmt + + + + + +

referenced by: +

+


alter_role_stmt:

+ + + + + + + ALTER + + + + role_or_group_or_user + + + + IF + + + EXISTS + + + + string_or_placeholder + + + + + opt_role_options + + + + + +

referenced by: +

+


partitioned_backup:

+ + + + + + + + string_or_placeholder + + + + ( + + + + string_or_placeholder_list + + + + ) + + + + +

referenced by: +

+


opt_as_of_clause:

+ + + + + + + + as_of_clause + + + + + +

referenced by: +

+


opt_incremental:

+ + + + + + + INCREMENTAL + + + FROM + + + + string_or_placeholder_list + + + + + +

referenced by: +

+


cancel_jobs_stmt:

+ + + + + + + CANCEL + + + JOB + + + + a_expr + + + + JOBS + + + + select_stmt + + + + + +

referenced by: +

+


cancel_queries_stmt:

+ + + + + + + CANCEL + + + QUERY + + + IF + + + EXISTS + + + + a_expr + + + + QUERIES + + + IF + + + EXISTS + + + + select_stmt + + + + + +

referenced by: +

+


cancel_sessions_stmt:

+ + + + + + + CANCEL + + + SESSION + + + IF + + + EXISTS + + + + a_expr + + + + SESSIONS + + + IF + + + EXISTS + + + + select_stmt + + + + + +

referenced by: +

+


create_role_stmt:

+ + + + + + + CREATE + + + + role_or_group_or_user + + + + IF + + + NOT + + + EXISTS + + + + string_or_placeholder + + + + + opt_role_options + + + + + +

referenced by: +

+


create_ddl_stmt:

+ + + + + + + + create_changefeed_stmt + + + + + create_database_stmt + + + + + create_index_stmt + + + + + create_schema_stmt + + + + + create_table_stmt + + + + + create_table_as_stmt + + + + + create_view_stmt + + + + + create_sequence_stmt + + + + + +

referenced by: +

+


create_stats_stmt:

+ + + + + + + CREATE + + + STATISTICS + + + + statistics_name + + + + + opt_stats_columns + + + + FROM + + + + create_stats_target + + + + + opt_create_stats_options + + + + + +

referenced by: +

+


opt_with_clause:

+ + + + + + + + with_clause + + + + + +

referenced by: +

+


table_expr_opt_alias_idx:

+ + + + + + + + table_name_opt_idx + + + + AS + + + + table_alias_name + + + + + +

referenced by: +

+


opt_where_clause:

+ + + + + + + + where_clause + + + + + +

referenced by: +

+


opt_sort_clause:

+ + + + + + + + sort_clause + + + + + +

referenced by: +

+


opt_limit_clause:

+ + + + + + + + limit_clause + + + + + +

referenced by: +

+


returning_clause:

+ + + + + + + RETURNING + + + + target_list + + + + NOTHING + + + + +

referenced by: +

+


drop_ddl_stmt:

+ + + + + + + + drop_database_stmt + + + + + drop_index_stmt + + + + + drop_table_stmt + + + + + drop_view_stmt + + + + + drop_sequence_stmt + + + + + +

referenced by: +

+


drop_role_stmt:

+ + + + + + + DROP + + + + role_or_group_or_user + + + + IF + + + EXISTS + + + + string_or_placeholder_list + + + + + +

referenced by: +

+


explain_option_list:

+ + + + + + + + explain_option_name + + + + , + + + + +

referenced by: +

+


import_format:

+ + + + + + + + name + + + + + +

referenced by: +

+


string_or_placeholder:

+ + + + + + + + non_reserved_word_or_sconst + + + + PLACEHOLDER + + + + +

referenced by: +

+


string_or_placeholder_list:

+ + + + + + + + string_or_placeholder + + + + , + + + + +

referenced by: +

+


table_elem_list:

+ + + + + + + + table_elem + + + + , + + + + +

referenced by: +

+


insert_column_list:

+ + + + + + + + insert_column_item + + + + , + + + + +

referenced by: +

+


insert_target:

+ + + + + + + + table_name + + + + AS + + + + table_alias_name + + + + + +

referenced by: +

+


insert_rest:

+ + + + + + + ( + + + + insert_column_list + + + + ) + + + + select_stmt + + + + DEFAULT + + + VALUES + + + + +

referenced by: +

+


on_conflict:

+ + + + + + + ON + + + CONFLICT + + + + opt_conf_expr + + + + DO + + + UPDATE + + + SET + + + + set_clause_list + + + + + opt_where_clause + + + + NOTHING + + + + +

referenced by: +

+


a_expr:

+ + + + + + + + c_expr + + + + + + + + - + + + ~ + + + NOT + + + + a_expr + + + + DEFAULT + + + TYPECAST + + + + cast_target + + + + TYPEANNOTATE + + + + typename + + + + COLLATE + + + + collation_name + + + + AT + + + TIME + + + ZONE + + + + a_expr + + + + + + + + + a_expr + + + + - + + + + a_expr + + + + * + + + + a_expr + + + + / + + + + a_expr + + + + FLOORDIV + + + + a_expr + + + + % + + + + a_expr + + + + ^ + + + + a_expr + + + + # + + + + a_expr + + + + & + + + + a_expr + + + + | + + + + a_expr + + + + < + + + + a_expr + + + + > + + + + a_expr + + + + ? + + + + a_expr + + + + JSON_SOME_EXISTS + + + + a_expr + + + + JSON_ALL_EXISTS + + + + a_expr + + + + CONTAINS + + + + a_expr + + + + CONTAINED_BY + + + + a_expr + + + + = + + + + a_expr + + + + CONCAT + + + + a_expr + + + + LSHIFT + + + + a_expr + + + + RSHIFT + + + + a_expr + + + + FETCHVAL + + + + a_expr + + + + FETCHTEXT + + + + a_expr + + + + FETCHVAL_PATH + + + + a_expr + + + + FETCHTEXT_PATH + + + + a_expr + + + + REMOVE_PATH + + + + a_expr + + + + INET_CONTAINED_BY_OR_EQUALS + + + + a_expr + + + + AND_AND + + + + a_expr + + + + INET_CONTAINS_OR_EQUALS + + + + a_expr + + + + LESS_EQUALS + + + + a_expr + + + + GREATER_EQUALS + + + + a_expr + + + + NOT_EQUALS + + + + a_expr + + + + AND + + + + a_expr + + + + OR + + + + a_expr + + + + LIKE + + + + a_expr + + + + LIKE + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + NOT + + + LIKE + + + + a_expr + + + + NOT + + + LIKE + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + ILIKE + + + + a_expr + + + + ILIKE + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + NOT + + + ILIKE + + + + a_expr + + + + NOT + + + ILIKE + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + SIMILAR + + + TO + + + + a_expr + + + + SIMILAR + + + TO + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + NOT + + + SIMILAR + + + TO + + + + a_expr + + + + NOT + + + SIMILAR + + + TO + + + + a_expr + + + + ESCAPE + + + + a_expr + + + + ~ + + + + a_expr + + + + NOT_REGMATCH + + + + a_expr + + + + REGIMATCH + + + + a_expr + + + + NOT_REGIMATCH + + + + a_expr + + + + IS + + + NAN + + + IS + + + NOT + + + NAN + + + IS + + + NULL + + + ISNULL + + + IS + + + NOT + + + NULL + + + NOTNULL + + + IS + + + TRUE + + + IS + + + NOT + + + TRUE + + + IS + + + FALSE + + + IS + + + NOT + + + FALSE + + + IS + + + UNKNOWN + + + IS + + + NOT + + + UNKNOWN + + + IS + + + DISTINCT + + + FROM + + + + a_expr + + + + IS + + + NOT + + + DISTINCT + + + FROM + + + + a_expr + + + + IS + + + OF + + + ( + + + + type_list + + + + ) + + + IS + + + NOT + + + OF + + + ( + + + + type_list + + + + ) + + + BETWEEN + + + + opt_asymmetric + + + + + b_expr + + + + AND + + + + a_expr + + + + NOT + + + BETWEEN + + + + opt_asymmetric + + + + + b_expr + + + + AND + + + + a_expr + + + + BETWEEN + + + SYMMETRIC + + + + b_expr + + + + AND + + + + a_expr + + + + NOT + + + BETWEEN + + + SYMMETRIC + + + + b_expr + + + + AND + + + + a_expr + + + + IN + + + + in_expr + + + + NOT + + + IN + + + + in_expr + + + + + subquery_op + + + + + sub_type + + + + + a_expr + + + + + +

referenced by: +

+


reset_session_stmt:

+ + + + + + + RESET + + + SESSION + + + + session_var + + + + + +

referenced by: +

+


reset_csetting_stmt:

+ + + + + + + RESET + + + CLUSTER + + + SETTING + + + + var_name + + + + + +

referenced by: +

+


partitioned_backup_list:

+ + + + + + + + partitioned_backup + + + + , + + + + +

referenced by: +

+


scrub_table_stmt:

+ + + + + + + EXPERIMENTAL + + + SCRUB + + + TABLE + + + + table_name + + + + + opt_as_of_clause + + + + + opt_scrub_options_clause + + + + + +

referenced by: +

+


scrub_database_stmt:

+ + + + + + + EXPERIMENTAL + + + SCRUB + + + DATABASE + + + + database_name + + + + + opt_as_of_clause + + + + + +

referenced by: +

+


select_no_parens:

+ + + + + + + + simple_select + + + + + select_clause + + + + + sort_clause + + + + + opt_sort_clause + + + + + for_locking_clause + + + + + opt_select_limit + + + + + select_limit + + + + + opt_for_locking_clause + + + + + with_clause + + + + + select_clause + + + + + sort_clause + + + + + opt_sort_clause + + + + + for_locking_clause + + + + + opt_select_limit + + + + + select_limit + + + + + opt_for_locking_clause + + + + + +

referenced by: +

+


select_with_parens:

+ + + + + + + ( + + + + select_no_parens + + + + + select_with_parens + + + + ) + + + + +

referenced by: +

+


set_session_stmt:

+ + + + + + + SET + + + SESSION + + + + set_rest_more + + + + CHARACTERISTICS + + + AS + + + TRANSACTION + + + + transaction_mode_list + + + + + set_rest_more + + + + + +

referenced by: +

+


set_csetting_stmt:

+ + + + + + + SET + + + CLUSTER + + + SETTING + + + + var_name + + + + + to_or_eq + + + + + var_value + + + + + +

referenced by: +

+


use_stmt:

+ + + + + + + USE + + + + var_value + + + + + +

referenced by: +

+


show_backup_stmt:

+ + + + + + + SHOW + + + BACKUP + + + SCHEMAS + + + + string_or_placeholder + + + + + opt_with_options + + + + + +

referenced by: +

+


show_columns_stmt:

+ + + + + + + SHOW + + + COLUMNS + + + FROM + + + + table_name + + + + + with_comment + + + + + +

referenced by: +

+


show_constraints_stmt:

+ + + + + + + SHOW + + + CONSTRAINT + + + CONSTRAINTS + + + FROM + + + + table_name + + + + + +

referenced by: +

+


show_create_stmt:

+ + + + + + + SHOW + + + CREATE + + + + table_name + + + + + +

referenced by: +

+


show_csettings_stmt:

+ + + + + + + SHOW + + + CLUSTER + + + SETTING + + + + var_name + + + + ALL + + + SETTINGS + + + ALL + + + PUBLIC + + + CLUSTER + + + SETTINGS + + + + +

referenced by: +

+


show_databases_stmt:

+ + + + + + + SHOW + + + DATABASES + + + + with_comment + + + + + +

referenced by: +

+


show_grants_stmt:

+ + + + + + + SHOW + + + GRANTS + + + + opt_on_targets_roles + + + + + for_grantee_clause + + + + + +

referenced by: +

+


show_indexes_stmt:

+ + + + + + + SHOW + + + INDEX + + + INDEXES + + + KEYS + + + FROM + + + + table_name + + + + DATABASE + + + + database_name + + + + + with_comment + + + + + +

referenced by: +

+


show_partitions_stmt:

+ + + + + + + SHOW + + + PARTITIONS + + + FROM + + + TABLE + + + + table_name + + + + DATABASE + + + + database_name + + + + INDEX + + + + table_index_name + + + + + table_name + + + + @ + + + * + + + + +

referenced by: +

+


show_jobs_stmt:

+ + + + + + + SHOW + + + AUTOMATIC + + + JOBS + + + JOBS + + + WHEN + + + COMPLETE + + + + select_stmt + + + + JOB + + + WHEN + + + COMPLETE + + + + a_expr + + + + + +

referenced by: +

+


show_queries_stmt:

+ + + + + + + SHOW + + + ALL + + + + opt_cluster + + + + QUERIES + + + + +

referenced by: +

+


show_ranges_stmt:

+ + + + + + + SHOW + + + RANGES + + + FROM + + + TABLE + + + + table_name + + + + INDEX + + + + table_index_name + + + + DATABASE + + + + database_name + + + + + +

referenced by: +

+


show_range_for_row_stmt:

+ + + + + + + SHOW + + + RANGE + + + FROM + + + TABLE + + + + table_name + + + + INDEX + + + + table_index_name + + + + FOR + + + ROW + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


show_roles_stmt:

+ + + + + + + SHOW + + + ROLES + + + + +

referenced by: +

+


show_savepoint_stmt:

+ + + + + + + SHOW + + + SAVEPOINT + + + STATUS + + + + +

referenced by: +

+


show_schemas_stmt:

+ + + + + + + SHOW + + + SCHEMAS + + + FROM + + + + name + + + + + +

referenced by: +

+


show_sequences_stmt:

+ + + + + + + SHOW + + + SEQUENCES + + + FROM + + + + name + + + + + +

referenced by: +

+


show_session_stmt:

+ + + + + + + SHOW + + + SESSION + + + + session_var + + + + + +

referenced by: +

+


show_sessions_stmt:

+ + + + + + + SHOW + + + ALL + + + + opt_cluster + + + + SESSIONS + + + + +

referenced by: +

+


show_stats_stmt:

+ + + + + + + SHOW + + + STATISTICS + + + FOR + + + TABLE + + + + table_name + + + + + +

referenced by: +

+


show_tables_stmt:

+ + + + + + + SHOW + + + TABLES + + + FROM + + + + name + + + + . + + + + name + + + + + with_comment + + + + + +

referenced by: +

+


show_trace_stmt:

+ + + + + + + SHOW + + + + opt_compact + + + + KV + + + TRACE + + + FOR + + + SESSION + + + + +

referenced by: +

+


show_users_stmt:

+ + + + + + + SHOW + + + USERS + + + + +

referenced by: +

+


show_zone_stmt:

+ + + + + + + SHOW + + + ZONE + + + CONFIGURATION + + + FOR + + + RANGE + + + + zone_name + + + + DATABASE + + + + database_name + + + + TABLE + + + + table_name + + + + INDEX + + + + table_index_name + + + + + opt_partition + + + + PARTITION + + + + partition_name + + + + OF + + + TABLE + + + + table_name + + + + INDEX + + + + table_index_name + + + + CONFIGURATIONS + + + ALL + + + ZONE + + + CONFIGURATIONS + + + + +

referenced by: +

+


opt_table:

+ + + + + + + TABLE + + + + +

referenced by: +

+


relation_expr_list:

+ + + + + + + + relation_expr + + + + , + + + + +

referenced by: +

+


opt_drop_behavior:

+ + + + + + + CASCADE + + + RESTRICT + + + + +

referenced by: +

+


set_clause_list:

+ + + + + + + + set_clause + + + + , + + + + +

referenced by: +

+


opt_from_list:

+ + + + + + + FROM + + + + from_list + + + + + +

referenced by: +

+


db_object_name:

+ + + + + + + + simple_db_object_name + + + + + complex_db_object_name + + + + + +

referenced by: +

+


kv_option_list:

+ + + + + + + + kv_option + + + + , + + + + +

referenced by: +

+


prefixed_column_path:

+ + + + + + + + db_object_name_component + + + + . + + + + unrestricted_name + + + + . + + + + unrestricted_name + + + + . + + + + unrestricted_name + + + + + +

referenced by: +

+


index_name:

+ + + + + + + + unrestricted_name + + + + + +

referenced by: +

+


standalone_index_name:

+ + + + + + + + db_object_name + + + + + +

referenced by: +

+


expr_list:

+ + + + + + + + a_expr + + + + , + + + + +

referenced by: +

+


unreserved_keyword:

+ + + + + + + ABORT + + + ACTION + + + ADD + + + ADMIN + + + AGGREGATE + + + ALTER + + + AT + + + AUTOMATIC + + + AUTHORIZATION + + + BACKUP + + + BEGIN + + + BIGSERIAL + + + BLOB + + + BOOL + + + BUCKET_COUNT + + + BUNDLE + + + BY + + + BYTEA + + + BYTES + + + CACHE + + + CANCEL + + + CASCADE + + + CHANGEFEED + + + CLUSTER + + + COLUMNS + + + COMMENT + + + COMMIT + + + COMMITTED + + + COMPACT + + + COMPLETE + + + CONFLICT + + + CONFIGURATION + + + CONFIGURATIONS + + + CONFIGURE + + + CONSTRAINTS + + + CONVERSION + + + COPY + + + COVERING + + + CREATEROLE + + + CUBE + + + CURRENT + + + CYCLE + + + DATA + + + DATABASE + + + DATABASES + + + DATE + + + DAY + + + DEALLOCATE + + + DELETE + + + DEFERRED + + + DISCARD + + + DOMAIN + + + DOUBLE + + + DROP + + + ENCODING + + + ENUM + + + ESCAPE + + + EXCLUDE + + + EXECUTE + + + EXPERIMENTAL + + + EXPERIMENTAL_AUDIT + + + EXPERIMENTAL_FINGERPRINTS + + + EXPERIMENTAL_RELOCATE + + + EXPERIMENTAL_REPLICA + + + EXPIRATION + + + EXPLAIN + + + EXPORT + + + EXTENSION + + + FILES + + + FILTER + + + FIRST + + + FLOAT4 + + + FLOAT8 + + + FOLLOWING + + + FORCE_INDEX + + + FUNCTION + + + GLOBAL + + + GRANTS + + + GROUPS + + + HASH + + + HIGH + + + HISTOGRAM + + + HOUR + + + IMMEDIATE + + + IMPORT + + + INCLUDE + + + INCREMENT + + + INCREMENTAL + + + INDEXES + + + INET + + + INJECT + + + INSERT + + + INT2 + + + INT2VECTOR + + + INT4 + + + INT8 + + + INT64 + + + INTERLEAVE + + + INVERTED + + + ISOLATION + + + JOB + + + JOBS + + + JSON + + + JSONB + + + KEY + + + KEYS + + + KV + + + LANGUAGE + + + LAST + + + LC_COLLATE + + + LC_CTYPE + + + LEASE + + + LESS + + + LEVEL + + + LIST + + + LOCAL + + + LOCKED + + + LOGIN + + + LOOKUP + + + LOW + + + MATCH + + + MATERIALIZED + + + MAXVALUE + + + MERGE + + + MINUTE + + + MINVALUE + + + MONTH + + + NAMES + + + NAN + + + NAME + + + NEXT + + + NO + + + NORMAL + + + NO_INDEX_JOIN + + + NOCREATEROLE + + + NOLOGIN + + + NOWAIT + + + NULLS + + + IGNORE_FOREIGN_KEYS + + + OF + + + OFF + + + OID + + + OIDS + + + OIDVECTOR + + + OPERATOR + + + OPT + + + OPTION + + + OPTIONS + + + ORDINALITY + + + OTHERS + + + OVER + + + OWNED + + + PARENT + + + PARTIAL + + + PARTITION + + + PARTITIONS + + + PASSWORD + + + PAUSE + + + PHYSICAL + + + PLAN + + + PLANS + + + PRECEDING + + + PREPARE + + + PRESERVE + + + PRIORITY + + + PUBLIC + + + PUBLICATION + + + QUERIES + + + QUERY + + + RANGE + + + RANGES + + + READ + + + RECURSIVE + + + REF + + + REGCLASS + + + REGPROC + + + REGPROCEDURE + + + REGNAMESPACE + + + REGTYPE + + + REINDEX + + + RELEASE + + + RENAME + + + REPEATABLE + + + REPLACE + + + RESET + + + RESTORE + + + RESTRICT + + + RESUME + + + REVOKE + + + ROLE + + + ROLES + + + ROLLBACK + + + ROLLUP + + + ROWS + + + RULE + + + SETTING + + + SETTINGS + + + STATUS + + + SAVEPOINT + + + SCATTER + + + SCHEMA + + + SCHEMAS + + + SCRUB + + + SEARCH + + + SECOND + + + SERIAL + + + SERIALIZABLE + + + SERIAL2 + + + SERIAL4 + + + SERIAL8 + + + SEQUENCE + + + SEQUENCES + + + SERVER + + + SESSION + + + SESSIONS + + + SET + + + SHARE + + + SHOW + + + SIMPLE + + + SKIP + + + SMALLSERIAL + + + SNAPSHOT + + + SPLIT + + + SQL + + + START + + + STATISTICS + + + STDIN + + + STORE + + + STORED + + + STORING + + + STRICT + + + STRING + + + SUBSCRIPTION + + + SYNTAX + + + SYSTEM + + + TABLES + + + TEMP + + + TEMPLATE + + + TEMPORARY + + + TESTING_RELOCATE + + + TEXT + + + TIES + + + TRACE + + + TRANSACTION + + + TRIGGER + + + TRUNCATE + + + TRUSTED + + + TYPE + + + THROTTLING + + + UNBOUNDED + + + UNCOMMITTED + + + UNKNOWN + + + UNLOGGED + + + UNSPLIT + + + UNTIL + + + UPDATE + + + UPSERT + + + UUID + + + USE + + + USERS + + + VALID + + + VALIDATE + + + VALUE + + + VARYING + + + VIEW + + + WITHIN + + + WITHOUT + + + WRITE + + + YEAR + + + ZONE + + + + +

referenced by: +

+


col_name_keyword:

+ + + + + + + ANNOTATE_TYPE + + + BETWEEN + + + BIGINT + + + BIT + + + BOOLEAN + + + CHAR + + + CHARACTER + + + CHARACTERISTICS + + + COALESCE + + + DEC + + + DECIMAL + + + EXISTS + + + EXTRACT + + + EXTRACT_DURATION + + + FLOAT + + + GREATEST + + + GROUPING + + + IF + + + IFERROR + + + IFNULL + + + INT + + + INTEGER + + + INTERVAL + + + ISERROR + + + LEAST + + + NULLIF + + + NUMERIC + + + OUT + + + OVERLAY + + + POSITION + + + PRECISION + + + REAL + + + ROW + + + SMALLINT + + + SUBSTRING + + + TIME + + + TIMETZ + + + TIMESTAMP + + + TIMESTAMPTZ + + + TREAT + + + TRIM + + + VALUES + + + VARBIT + + + VARCHAR + + + VIRTUAL + + + WORK + + + + +

referenced by: +

+


complex_table_pattern:

+ + + + + + + + complex_db_object_name + + + + + db_object_name_component + + + + . + + + + unrestricted_name + + + + . + + + * + + + + +

referenced by: +

+


table_pattern:

+ + + + + + + + simple_db_object_name + + + + + complex_table_pattern + + + + + +

referenced by: +

+


table_pattern_list:

+ + + + + + + + table_pattern + + + + , + + + + +

referenced by: +

+


privilege:

+ + + + + + + + name + + + + CREATE + + + GRANT + + + SELECT + + + + +

referenced by: +

+


type_list:

+ + + + + + + + typename + + + + , + + + + +

referenced by: +

+


transaction_mode_list:

+ + + + + + + + transaction_mode + + + + + opt_comma + + + + + +

referenced by: +

+


opt_transaction:

+ + + + + + + TRANSACTION + + + + +

referenced by: +

+


begin_transaction:

+ + + + + + + + transaction_mode_list + + + + + +

referenced by: +

+


opt_abort_mod:

+ + + + + + + TRANSACTION + + + WORK + + + + +

referenced by: +

+


alter_table_stmt:

+ + + + + + + + alter_onetable_stmt + + + + + alter_split_stmt + + + + + alter_unsplit_stmt + + + + + alter_scatter_stmt + + + + + alter_zone_table_stmt + + + + + alter_rename_table_stmt + + + + + +

referenced by: +

+


alter_index_stmt:

+ + + + + + + + alter_oneindex_stmt + + + + + alter_split_index_stmt + + + + + alter_unsplit_index_stmt + + + + + alter_scatter_index_stmt + + + + + alter_rename_index_stmt + + + + + alter_zone_index_stmt + + + + + +

referenced by: +

+


alter_view_stmt:

+ + + + + + + + alter_rename_view_stmt + + + + + +

referenced by: +

+


alter_sequence_stmt:

+ + + + + + + + alter_rename_sequence_stmt + + + + + alter_sequence_options_stmt + + + + + +

referenced by: +

+


alter_database_stmt:

+ + + + + + + + alter_rename_database_stmt + + + + + alter_zone_database_stmt + + + + + +

referenced by: +

+


alter_range_stmt:

+ + + + + + + + alter_zone_range_stmt + + + + + +

referenced by: +

+


alter_partition_stmt:

+ + + + + + + + alter_zone_partition_stmt + + + + + +

referenced by: +

+


role_or_group_or_user:

+ + + + + + + ROLE + + + USER + + + + +

referenced by: +

+


opt_role_options:

+ + + + + + + + opt_with + + + + + role_options + + + + + +

referenced by: +

+


as_of_clause:

+ + + + + + + AS + + + OF + + + SYSTEM + + + TIME + + + + a_expr + + + + + +

referenced by: +

+


create_changefeed_stmt:

+ + + + + + + CREATE + + + CHANGEFEED + + + FOR + + + + changefeed_targets + + + + + opt_changefeed_sink + + + + + opt_with_options + + + + + +

referenced by: +

+


create_database_stmt:

+ + + + + + + CREATE + + + DATABASE + + + IF + + + NOT + + + EXISTS + + + + database_name + + + + + opt_with + + + + + opt_template_clause + + + + + opt_encoding_clause + + + + + opt_lc_collate_clause + + + + + opt_lc_ctype_clause + + + + + +

referenced by: +

+


create_index_stmt:

+ + + + + + + CREATE + + + + opt_unique + + + + INDEX + + + + opt_concurrently + + + + + opt_index_name + + + + IF + + + NOT + + + EXISTS + + + + index_name + + + + ON + + + + table_name + + + + + opt_using_gin_btree + + + + ( + + + + index_params + + + + ) + + + + opt_hash_sharded + + + + INVERTED + + + INDEX + + + + opt_concurrently + + + + + opt_index_name + + + + IF + + + NOT + + + EXISTS + + + + index_name + + + + ON + + + + table_name + + + + ( + + + + index_params + + + + ) + + + + opt_storing + + + + + opt_interleave + + + + + opt_partition_by + + + + + +

referenced by: +

+


create_schema_stmt:

+ + + + + + + CREATE + + + SCHEMA + + + IF + + + NOT + + + EXISTS + + + + schema_name + + + + + +

referenced by: +

+


create_table_stmt:

+ + + + + + + CREATE + + + + opt_temp_create_table + + + + TABLE + + + IF + + + NOT + + + EXISTS + + + + table_name + + + + ( + + + + opt_table_elem_list + + + + ) + + + + opt_interleave + + + + + opt_partition_by + + + + + +

referenced by: +

+


create_table_as_stmt:

+ + + + + + + CREATE + + + + opt_temp_create_table + + + + TABLE + + + IF + + + NOT + + + EXISTS + + + + table_name + + + + + create_as_opt_col_list + + + + AS + + + + select_stmt + + + + + +

referenced by: +

+


create_view_stmt:

+ + + + + + + CREATE + + + + opt_temp + + + + VIEW + + + IF + + + NOT + + + EXISTS + + + + view_name + + + + + opt_column_list + + + + AS + + + + select_stmt + + + + + +

referenced by: +

+


create_sequence_stmt:

+ + + + + + + CREATE + + + + opt_temp + + + + SEQUENCE + + + IF + + + NOT + + + EXISTS + + + + sequence_name + + + + + opt_sequence_option_list + + + + + +

referenced by: +

+


statistics_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


opt_stats_columns:

+ + + + + + + ON + + + + name_list + + + + + +

referenced by: +

+


create_stats_target:

+ + + + + + + + table_name + + + + + +

referenced by: +

+


opt_create_stats_options:

+ + + + + + + + as_of_clause + + + + + +

referenced by: +

+


with_clause:

+ + + + + + + WITH + + + RECURSIVE + + + + cte_list + + + + + +

referenced by: +

+


table_name_opt_idx:

+ + + + + + + + table_name + + + + + opt_index_flags + + + + + +

referenced by: +

+


where_clause:

+ + + + + + + WHERE + + + + a_expr + + + + + +

referenced by: +

+


sort_clause:

+ + + + + + + ORDER + + + BY + + + + sortby_list + + + + + +

referenced by: +

+


limit_clause:

+ + + + + + + LIMIT + + + ALL + + + + a_expr + + + + FETCH + + + + first_or_next + + + + + select_fetch_first_value + + + + + row_or_rows + + + + ONLY + + + + +

referenced by: +

+


target_list:

+ + + + + + + + target_elem + + + + , + + + + +

referenced by: +

+


drop_database_stmt:

+ + + + + + + DROP + + + DATABASE + + + IF + + + EXISTS + + + + database_name + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_index_stmt:

+ + + + + + + DROP + + + INDEX + + + + opt_concurrently + + + + IF + + + EXISTS + + + + table_index_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_table_stmt:

+ + + + + + + DROP + + + TABLE + + + IF + + + EXISTS + + + + table_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_view_stmt:

+ + + + + + + DROP + + + VIEW + + + IF + + + EXISTS + + + + table_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


drop_sequence_stmt:

+ + + + + + + DROP + + + SEQUENCE + + + IF + + + EXISTS + + + + table_name_list + + + + + opt_drop_behavior + + + + + +

referenced by: +

+


explain_option_name:

+ + + + + + + + non_reserved_word + + + + + +

referenced by: +

+


non_reserved_word_or_sconst:

+ + + + + + + + non_reserved_word + + + + SCONST + + + + +

referenced by: +

+


table_elem:

+ + + + + + + + column_def + + + + + index_def + + + + + family_def + + + + + table_constraint + + + + + +

referenced by: +

+


insert_column_item:

+ + + + + + + + column_name + + + + + +

referenced by: +

+


opt_conf_expr:

+ + + + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


c_expr:

+ + + + + + + + d_expr + + + + + array_subscripts + + + + + case_expr + + + + EXISTS + + + + select_with_parens + + + + + +

referenced by: +

+


cast_target:

+ + + + + + + + typename + + + + + +

referenced by: +

+


typename:

+ + + + + + + + simple_typename + + + + + opt_array_bounds + + + + ARRAY + + + + +

referenced by: +

+


collation_name:

+ + + + + + + + unrestricted_name + + + + + +

referenced by: +

+


opt_asymmetric:

+ + + + + + + ASYMMETRIC + + + + +

referenced by: +

+


b_expr:

+ + + + + + + + c_expr + + + + + + + + - + + + ~ + + + + b_expr + + + + TYPECAST + + + + cast_target + + + + TYPEANNOTATE + + + + typename + + + + + + + + - + + + * + + + / + + + FLOORDIV + + + % + + + ^ + + + # + + + & + + + | + + + < + + + > + + + = + + + CONCAT + + + LSHIFT + + + RSHIFT + + + LESS_EQUALS + + + GREATER_EQUALS + + + NOT_EQUALS + + + + b_expr + + + + IS + + + NOT + + + DISTINCT + + + FROM + + + + b_expr + + + + OF + + + ( + + + + type_list + + + + ) + + + + +

referenced by: +

+


in_expr:

+ + + + + + + + select_with_parens + + + + + expr_tuple1_ambiguous + + + + + +

referenced by: +

+


subquery_op:

+ + + + + + + + math_op + + + + NOT + + + LIKE + + + ILIKE + + + + +

referenced by: +

+


sub_type:

+ + + + + + + ANY + + + SOME + + + ALL + + + + +

referenced by: +

+


session_var:

+ + + + + + + identifier + + + ALL + + + DATABASE + + + NAMES + + + SESSION_USER + + + TIME + + + ZONE + + + + +

referenced by: +

+


var_name:

+ + + + + + + + name + + + + + attrs + + + + + +

referenced by: +

+


opt_scrub_options_clause:

+ + + + + + + WITH + + + OPTIONS + + + + scrub_option_list + + + + + +

referenced by: +

+


simple_select:

+ + + + + + + + simple_select_clause + + + + + values_clause + + + + + table_clause + + + + + set_operation + + + + + +

referenced by: +

+


select_clause:

+ + + + + + + + simple_select + + + + + select_with_parens + + + + + +

referenced by: +

+


for_locking_clause:

+ + + + + + + + for_locking_items + + + + FOR + + + READ + + + ONLY + + + + +

referenced by: +

+


opt_select_limit:

+ + + + + + + + select_limit + + + + + +

referenced by: +

+


select_limit:

+ + + + + + + + limit_clause + + + + + offset_clause + + + + + offset_clause + + + + + limit_clause + + + + + +

referenced by: +

+


opt_for_locking_clause:

+ + + + + + + + for_locking_clause + + + + + +

referenced by: +

+


set_rest_more:

+ + + + + + + + generic_set + + + + + +

referenced by: +

+


to_or_eq:

+ + + + + + + = + + + TO + + + + +

referenced by: +

+


var_value:

+ + + + + + + + a_expr + + + + + extra_var_value + + + + + +

referenced by: +

+


with_comment:

+ + + + + + + WITH + + + COMMENT + + + + +

referenced by: +

+


opt_on_targets_roles:

+ + + + + + + ON + + + + targets_roles + + + + + +

referenced by: +

+


for_grantee_clause:

+ + + + + + + FOR + + + + name_list + + + + + +

referenced by: +

+


opt_cluster:

+ + + + + + + CLUSTER + + + LOCAL + + + + +

referenced by: +

+


opt_compact:

+ + + + + + + COMPACT + + + + +

referenced by: +

+


zone_name:

+ + + + + + + + unrestricted_name + + + + + +

referenced by: +

+


opt_partition:

+ + + + + + + + partition + + + + + +

referenced by: +

+


partition_name:

+ + + + + + + + unrestricted_name + + + + + +

referenced by: +

+


relation_expr:

+ + + + + + + + table_name + + + + * + + + ONLY + + + + table_name + + + + ( + + + + table_name + + + + ) + + + + +

referenced by: +

+


set_clause:

+ + + + + + + + single_set_clause + + + + + multiple_set_clause + + + + + +

referenced by: +

+


from_list:

+ + + + + + + + table_ref + + + + , + + + + +

referenced by: +

+


simple_db_object_name:

+ + + + + + + + db_object_name_component + + + + + +

referenced by: +

+


complex_db_object_name:

+ + + + + + + + db_object_name_component + + + + . + + + + unrestricted_name + + + + . + + + + unrestricted_name + + + + + +

referenced by: +

+


kv_option:

+ + + + + + + + name + + + + SCONST + + + = + + + + string_or_placeholder + + + + + +

referenced by: +

+


db_object_name_component:

+ + + + + + + + name + + + + FAMILY + + + + cockroachdb_extra_reserved_keyword + + + + + +

referenced by: +

+


unrestricted_name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + type_func_name_keyword + + + + + reserved_keyword + + + + + +

referenced by: +

+


transaction_mode:

+ + + + + + + + transaction_user_priority + + + + + transaction_read_mode + + + + + as_of_clause + + + + + +

referenced by: +

+


opt_comma:

+ + + + + + + , + + + + +

referenced by: +

+


alter_onetable_stmt:

+ + + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + relation_expr + + + + + alter_table_cmds + + + + + +

referenced by: +

+


alter_split_stmt:

+ + + + + + + ALTER + + + TABLE + + + + table_name + + + + SPLIT + + + AT + + + + select_stmt + + + + WITH + + + EXPIRATION + + + + a_expr + + + + + +

referenced by: +

+


alter_unsplit_stmt:

+ + + + + + + ALTER + + + TABLE + + + + table_name + + + + UNSPLIT + + + AT + + + + select_stmt + + + + ALL + + + + +

referenced by: +

+


alter_scatter_stmt:

+ + + + + + + ALTER + + + TABLE + + + + table_name + + + + SCATTER + + + FROM + + + ( + + + + expr_list + + + + ) + + + TO + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


alter_zone_table_stmt:

+ + + + + + + ALTER + + + TABLE + + + + table_name + + + + + set_zone_config + + + + + +

referenced by: +

+


alter_rename_table_stmt:

+ + + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + relation_expr + + + + RENAME + + + TO + + + + table_name + + + + + +

referenced by: +

+


alter_oneindex_stmt:

+ + + + + + + ALTER + + + INDEX + + + IF + + + EXISTS + + + + table_index_name + + + + + alter_index_cmds + + + + + +

referenced by: +

+


alter_split_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_index_name + + + + SPLIT + + + AT + + + + select_stmt + + + + WITH + + + EXPIRATION + + + + a_expr + + + + + +

referenced by: +

+


alter_unsplit_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_index_name + + + + UNSPLIT + + + AT + + + + select_stmt + + + + ALL + + + + +

referenced by: +

+


alter_scatter_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_index_name + + + + SCATTER + + + FROM + + + ( + + + + expr_list + + + + ) + + + TO + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


alter_rename_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + IF + + + EXISTS + + + + table_index_name + + + + RENAME + + + TO + + + + index_name + + + + + +

referenced by: +

+


alter_zone_index_stmt:

+ + + + + + + ALTER + + + INDEX + + + + table_index_name + + + + + set_zone_config + + + + + +

referenced by: +

+


alter_rename_view_stmt:

+ + + + + + + ALTER + + + VIEW + + + IF + + + EXISTS + + + + relation_expr + + + + RENAME + + + TO + + + + view_name + + + + + +

referenced by: +

+


alter_rename_sequence_stmt:

+ + + + + + + ALTER + + + SEQUENCE + + + IF + + + EXISTS + + + + relation_expr + + + + RENAME + + + TO + + + + sequence_name + + + + + +

referenced by: +

+


alter_sequence_options_stmt:

+ + + + + + + ALTER + + + SEQUENCE + + + IF + + + EXISTS + + + + sequence_name + + + + + sequence_option_list + + + + + +

referenced by: +

+


alter_rename_database_stmt:

+ + + + + + + ALTER + + + DATABASE + + + + database_name + + + + RENAME + + + TO + + + + database_name + + + + + +

referenced by: +

+


alter_zone_database_stmt:

+ + + + + + + ALTER + + + DATABASE + + + + database_name + + + + + set_zone_config + + + + + +

referenced by: +

+


alter_zone_range_stmt:

+ + + + + + + ALTER + + + RANGE + + + + zone_name + + + + + set_zone_config + + + + + +

referenced by: +

+


alter_zone_partition_stmt:

+ + + + + + + ALTER + + + PARTITION + + + + partition_name + + + + OF + + + TABLE + + + + table_name + + + + INDEX + + + + table_index_name + + + + + table_name + + + + @ + + + * + + + + set_zone_config + + + + + +

referenced by: +

+


opt_with:

+ + + + + + + WITH + + + + +

referenced by: +

+


role_options:

+ + + + + + + + role_option + + + + + +

referenced by: +

+


changefeed_targets:

+ + + + + + + TABLE + + + + single_table_pattern_list + + + + + +

referenced by: +

+


opt_changefeed_sink:

+ + + + + + + INTO + + + + string_or_placeholder + + + + + +

referenced by: +

+


opt_template_clause:

+ + + + + + + TEMPLATE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_encoding_clause:

+ + + + + + + ENCODING + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_lc_collate_clause:

+ + + + + + + LC_COLLATE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_lc_ctype_clause:

+ + + + + + + LC_CTYPE + + + + opt_equal + + + + + non_reserved_word_or_sconst + + + + + +

referenced by: +

+


opt_unique:

+ + + + + + + UNIQUE + + + + +

referenced by: +

+


opt_concurrently:

+ + + + + + + CONCURRENTLY + + + + +

referenced by: +

+


opt_index_name:

+ + + + + + + + opt_name + + + + + +

referenced by: +

+


opt_using_gin_btree:

+ + + + + + + USING + + + + name + + + + + +

referenced by: +

+


index_params:

+ + + + + + + + index_elem + + + + , + + + + +

referenced by: +

+


opt_hash_sharded:

+ + + + + + + USING + + + HASH + + + WITH + + + BUCKET_COUNT + + + = + + + + a_expr + + + + + +

referenced by: +

+


opt_storing:

+ + + + + + + + storing + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_interleave:

+ + + + + + + INTERLEAVE + + + IN + + + PARENT + + + + table_name + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


opt_partition_by:

+ + + + + + + + partition_by + + + + + +

referenced by: +

+


schema_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


opt_temp_create_table:

+ + + + + + + + opt_temp + + + + LOCAL + + + GLOBAL + + + TEMPORARY + + + TEMP + + + + +

referenced by: +

+


opt_table_elem_list:

+ + + + + + + + table_elem_list + + + + + +

referenced by: +

+


create_as_opt_col_list:

+ + + + + + + ( + + + + create_as_table_defs + + + + ) + + + + +

referenced by: +

+


opt_temp:

+ + + + + + + TEMPORARY + + + TEMP + + + + +

referenced by: +

+


view_name:

+ + + + + + + + table_name + + + + + +

referenced by: +

+


sequence_name:

+ + + + + + + + db_object_name + + + + + +

referenced by: +

+


opt_sequence_option_list:

+ + + + + + + + sequence_option_list + + + + + +

referenced by: +

+


cte_list:

+ + + + + + + + common_table_expr + + + + , + + + + +

referenced by: +

+


opt_index_flags:

+ + + + + + + @ + + + + index_name + + + + [ + + + ICONST + + + ] + + + { + + + + index_flags_param_list + + + + } + + + + +

referenced by: +

+


sortby_list:

+ + + + + + + + sortby + + + + , + + + + +

referenced by: +

+


first_or_next:

+ + + + + + + FIRST + + + NEXT + + + + +

referenced by: +

+


select_fetch_first_value:

+ + + + + + + + c_expr + + + + + only_signed_iconst + + + + + only_signed_fconst + + + + + +

referenced by: +

+


row_or_rows:

+ + + + + + + ROW + + + ROWS + + + + +

referenced by: +

+


target_elem:

+ + + + + + + + a_expr + + + + AS + + + + target_name + + + + identifier + + + * + + + + +

referenced by: +

+


table_index_name_list:

+ + + + + + + + table_index_name + + + + , + + + + +

referenced by: +

+


table_name_list:

+ + + + + + + + table_name + + + + , + + + + +

referenced by: +

+


non_reserved_word:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + col_name_keyword + + + + + type_func_name_keyword + + + + + +

referenced by: +

+


column_def:

+ + + + + + + + column_name + + + + + typename + + + + + col_qual_list + + + + + +

referenced by: +

+


index_def:

+ + + + + + + UNIQUE + + + INDEX + + + + opt_index_name + + + + ( + + + + index_params + + + + ) + + + + opt_hash_sharded + + + + + opt_storing + + + + + opt_interleave + + + + + opt_partition_by + + + + INVERTED + + + INDEX + + + + opt_name + + + + ( + + + + index_params + + + + ) + + + + +

referenced by: +

+


family_def:

+ + + + + + + FAMILY + + + + opt_family_name + + + + ( + + + + name_list + + + + ) + + + + +

referenced by: +

+


table_constraint:

+ + + + + + + CONSTRAINT + + + + constraint_name + + + + + constraint_elem + + + + + +

referenced by: +

+


column_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


d_expr:

+ + + + + + + @ + + + ICONST + + + FCONST + + + + const_typename + + + + SCONST + + + BCONST + + + BITCONST + + + + interval_value + + + + TRUE + + + FALSE + + + NULL + + + + column_path_with_star + + + + PLACEHOLDER + + + ( + + + + a_expr + + + + ) + + + . + + + * + + + + unrestricted_name + + + + @ + + + ICONST + + + + func_expr + + + + + select_with_parens + + + + + labeled_row + + + + ARRAY + + + + select_with_parens + + + + + row + + + + + array_expr + + + + + +

referenced by: +

+


array_subscripts:

+ + + + + + + + array_subscript + + + + + +

referenced by: +

+


case_expr:

+ + + + + + + CASE + + + + case_arg + + + + + when_clause_list + + + + + case_default + + + + END + + + + +

referenced by: +

+


simple_typename:

+ + + + + + + + const_typename + + + + + bit_with_length + + + + + character_with_length + + + + + interval_type + + + + + postgres_oid + + + + + +

referenced by: +

+


opt_array_bounds:

+ + + + + + + [ + + + ] + + + + +

referenced by: +

+


expr_tuple1_ambiguous:

+ + + + + + + ( + + + + tuple1_ambiguous_values + + + + ) + + + + +

referenced by: +

+


math_op:

+ + + + + + + + + + + - + + + * + + + / + + + FLOORDIV + + + % + + + & + + + | + + + ^ + + + # + + + < + + + > + + + = + + + LESS_EQUALS + + + GREATER_EQUALS + + + NOT_EQUALS + + + + +

referenced by: +

+


attrs:

+ + + + + + + . + + + + unrestricted_name + + + + + +

referenced by: +

+


scrub_option_list:

+ + + + + + + + scrub_option + + + + , + + + + +

referenced by: +

+


simple_select_clause:

+ + + + + + + SELECT + + + + opt_all_clause + + + + DISTINCT + + + + distinct_on_clause + + + + + target_list + + + + + from_clause + + + + + opt_where_clause + + + + + group_clause + + + + + having_clause + + + + + window_clause + + + + + +

referenced by: +

+


values_clause:

+ + + + + + + VALUES + + + ( + + + + expr_list + + + + ) + + + , + + + + +

referenced by: +

+


table_clause:

+ + + + + + + TABLE + + + + table_ref + + + + + +

referenced by: +

+


set_operation:

+ + + + + + + + select_clause + + + + UNION + + + INTERSECT + + + EXCEPT + + + + all_or_distinct + + + + + select_clause + + + + + +

referenced by: +

+


for_locking_items:

+ + + + + + + + for_locking_item + + + + + +

referenced by: +

+


offset_clause:

+ + + + + + + OFFSET + + + + a_expr + + + + + select_fetch_first_value + + + + + row_or_rows + + + + + +

referenced by: +

+


generic_set:

+ + + + + + + + var_name + + + + + to_or_eq + + + + + var_list + + + + + +

referenced by: +

+


extra_var_value:

+ + + + + + + ON + + + + cockroachdb_extra_reserved_keyword + + + + + +

referenced by: +

+


targets_roles:

+ + + + + + + ROLE + + + + name_list + + + + + targets + + + + + +

referenced by: +

+


partition:

+ + + + + + + PARTITION + + + + partition_name + + + + + +

referenced by: +

+


single_set_clause:

+ + + + + + + + column_name + + + + = + + + + a_expr + + + + + +

referenced by: +

+


multiple_set_clause:

+ + + + + + + ( + + + + insert_column_list + + + + ) + + + = + + + + in_expr + + + + + +

referenced by: +

+


table_ref:

+ + + + + + + + relation_expr + + + + + opt_index_flags + + + + LATERAL + + + + select_with_parens + + + + + func_table + + + + [ + + + + row_source_extension_stmt + + + + ] + + + + opt_ordinality + + + + + opt_alias_clause + + + + + joined_table + + + + ( + + + + joined_table + + + + ) + + + + opt_ordinality + + + + + alias_clause + + + + + +

referenced by: +

+


cockroachdb_extra_reserved_keyword:

+ + + + + + + INDEX + + + NOTHING + + + + +

referenced by: +

+


type_func_name_keyword:

+ + + + + + + COLLATION + + + CROSS + + + FULL + + + INNER + + + ILIKE + + + IS + + + ISNULL + + + JOIN + + + LEFT + + + LIKE + + + NATURAL + + + NONE + + + NOTNULL + + + OUTER + + + OVERLAPS + + + RIGHT + + + SIMILAR + + + FAMILY + + + + +

referenced by: +

+


reserved_keyword:

+ + + + + + + ALL + + + ANALYSE + + + ANALYZE + + + AND + + + ANY + + + ARRAY + + + AS + + + ASC + + + ASYMMETRIC + + + BOTH + + + CASE + + + CAST + + + CHECK + + + COLLATE + + + COLUMN + + + CONCURRENTLY + + + CONSTRAINT + + + CREATE + + + CURRENT_CATALOG + + + CURRENT_DATE + + + CURRENT_ROLE + + + CURRENT_SCHEMA + + + CURRENT_TIME + + + CURRENT_TIMESTAMP + + + CURRENT_USER + + + DEFAULT + + + DEFERRABLE + + + DESC + + + DISTINCT + + + DO + + + ELSE + + + END + + + EXCEPT + + + FALSE + + + FETCH + + + FOR + + + FOREIGN + + + FROM + + + GRANT + + + GROUP + + + HAVING + + + IN + + + INITIALLY + + + INTERSECT + + + INTO + + + LATERAL + + + LEADING + + + LIMIT + + + LOCALTIME + + + LOCALTIMESTAMP + + + NOT + + + NULL + + + OFFSET + + + ON + + + ONLY + + + OR + + + ORDER + + + PLACING + + + PRIMARY + + + REFERENCES + + + RETURNING + + + SELECT + + + SESSION_USER + + + SOME + + + SYMMETRIC + + + TABLE + + + THEN + + + TO + + + TRAILING + + + TRUE + + + UNION + + + UNIQUE + + + USER + + + USING + + + VARIADIC + + + WHEN + + + WHERE + + + WINDOW + + + WITH + + + + cockroachdb_extra_reserved_keyword + + + + + +

referenced by: +

+


transaction_user_priority:

+ + + + + + + PRIORITY + + + + user_priority + + + + + +

referenced by: +

+


transaction_read_mode:

+ + + + + + + READ + + + ONLY + + + WRITE + + + + +

referenced by: +

+


alter_table_cmds:

+ + + + + + + + alter_table_cmd + + + + , + + + + +

referenced by: +

+


set_zone_config:

+ + + + + + + CONFIGURE + + + ZONE + + + USING + + + + var_set_list + + + + DISCARD + + + + +

referenced by: +

+


alter_index_cmds:

+ + + + + + + + alter_index_cmd + + + + , + + + + +

referenced by: +

+


sequence_option_list:

+ + + + + + + + sequence_option_elem + + + + + +

referenced by: +

+


role_option:

+ + + + + + + CREATEROLE + + + NOCREATEROLE + + + LOGIN + + + NOLOGIN + + + + password_clause + + + + + valid_until_clause + + + + + +

referenced by: +

+


single_table_pattern_list:

+ + + + + + + + table_name + + + + , + + + + +

referenced by: +

+


opt_equal:

+ + + + + + + = + + + + +

referenced by: +

+


opt_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


index_elem:

+ + + + + + + + a_expr + + + + + opt_asc_desc + + + + + opt_nulls_order + + + + + +

referenced by: +

+


storing:

+ + + + + + + COVERING + + + STORING + + + INCLUDE + + + + +

referenced by: +

+


partition_by:

+ + + + + + + PARTITION + + + BY + + + LIST + + + ( + + + + name_list + + + + ) + + + ( + + + + list_partitions + + + + RANGE + + + ( + + + + name_list + + + + ) + + + ( + + + + range_partitions + + + + ) + + + NOTHING + + + + +

referenced by: +

+


create_as_table_defs:

+ + + + + + + + column_name + + + + + create_as_col_qual_list + + + + , + + + + column_name + + + + + create_as_col_qual_list + + + + + family_def + + + + + create_as_constraint_def + + + + + +

referenced by: +

+


common_table_expr:

+ + + + + + + + table_alias_name + + + + + opt_column_list + + + + AS + + + ( + + + + preparable_stmt + + + + ) + + + + +

referenced by: +

+


index_flags_param_list:

+ + + + + + + + index_flags_param + + + + , + + + + +

referenced by: +

+


sortby:

+ + + + + + + + a_expr + + + + + opt_asc_desc + + + + + opt_nulls_order + + + + PRIMARY + + + KEY + + + + table_name + + + + INDEX + + + + table_name + + + + @ + + + + index_name + + + + + opt_asc_desc + + + + + +

referenced by: +

+


only_signed_iconst:

+ + + + + + + + + + + - + + + ICONST + + + + +

referenced by: +

+


only_signed_fconst:

+ + + + + + + + + + + - + + + FCONST + + + + +

referenced by: +

+


target_name:

+ + + + + + + + unrestricted_name + + + + + +

referenced by: +

+


col_qual_list:

+ + + + + + + + col_qualification + + + + + +

referenced by: +

+


opt_family_name:

+ + + + + + + + opt_name + + + + + +

referenced by: +

+


constraint_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


constraint_elem:

+ + + + + + + CHECK + + + ( + + + + a_expr + + + + ) + + + UNIQUE + + + ( + + + + index_params + + + + ) + + + + opt_storing + + + + + opt_interleave + + + + + opt_partition_by + + + + PRIMARY + + + KEY + + + ( + + + + index_params + + + + ) + + + + opt_hash_sharded + + + + + opt_interleave + + + + FOREIGN + + + KEY + + + ( + + + + name_list + + + + ) + + + REFERENCES + + + + table_name + + + + + opt_column_list + + + + + key_match + + + + + reference_actions + + + + + +

referenced by: +

+


const_typename:

+ + + + + + + + numeric + + + + + bit_without_length + + + + + character_without_length + + + + + const_datetime + + + + + const_json + + + + BLOB + + + BYTES + + + BYTEA + + + TEXT + + + NAME + + + SERIAL + + + SERIAL2 + + + SMALLSERIAL + + + SERIAL4 + + + SERIAL8 + + + BIGSERIAL + + + UUID + + + INET + + + OID + + + OIDVECTOR + + + INT2VECTOR + + + identifier + + + + +

referenced by: +

+


interval_value:

+ + + + + + + INTERVAL + + + SCONST + + + + opt_interval_qualifier + + + + ( + + + ICONST + + + ) + + + SCONST + + + + +

referenced by: +

+


column_path_with_star:

+ + + + + + + + column_path + + + + + db_object_name_component + + + + . + + + + unrestricted_name + + + + . + + + + unrestricted_name + + + + . + + + * + + + + +

referenced by: +

+


func_expr:

+ + + + + + + + func_application + + + + + filter_clause + + + + + over_clause + + + + + func_expr_common_subexpr + + + + + +

referenced by: +

+


labeled_row:

+ + + + + + + + row + + + + ( + + + + row + + + + AS + + + + name_list + + + + ) + + + + +

referenced by: +

+


row:

+ + + + + + + ROW + + + ( + + + + opt_expr_list + + + + ) + + + + expr_tuple_unambiguous + + + + + +

referenced by: +

+


array_expr:

+ + + + + + + [ + + + + opt_expr_list + + + + + array_expr_list + + + + ] + + + + +

referenced by: +

+


array_subscript:

+ + + + + + + [ + + + + a_expr + + + + + opt_slice_bound + + + + : + + + + opt_slice_bound + + + + ] + + + + +

referenced by: +

+


case_arg:

+ + + + + + + + a_expr + + + + + +

referenced by: +

+


when_clause_list:

+ + + + + + + + when_clause + + + + + +

referenced by: +

+


case_default:

+ + + + + + + ELSE + + + + a_expr + + + + + +

referenced by: +

+


bit_with_length:

+ + + + + + + BIT + + + + opt_varying + + + + VARBIT + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


character_with_length:

+ + + + + + + + character_base + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


interval_type:

+ + + + + + + INTERVAL + + + + interval_qualifier + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


postgres_oid:

+ + + + + + + REGPROC + + + REGPROCEDURE + + + REGCLASS + + + REGTYPE + + + REGNAMESPACE + + + + +

referenced by: +

+


tuple1_ambiguous_values:

+ + + + + + + + a_expr + + + + , + + + + expr_list + + + + + +

referenced by: +

+


scrub_option:

+ + + + + + + INDEX + + + CONSTRAINT + + + ALL + + + ( + + + + name_list + + + + ) + + + PHYSICAL + + + + +

referenced by: +

+


opt_all_clause:

+ + + + + + + ALL + + + + +

referenced by: +

+


from_clause:

+ + + + + + + FROM + + + + from_list + + + + + opt_as_of_clause + + + + + +

referenced by: +

+


group_clause:

+ + + + + + + GROUP + + + BY + + + + expr_list + + + + + +

referenced by: +

+


having_clause:

+ + + + + + + HAVING + + + + a_expr + + + + + +

referenced by: +

+


window_clause:

+ + + + + + + WINDOW + + + + window_definition_list + + + + + +

referenced by: +

+


distinct_on_clause:

+ + + + + + + DISTINCT + + + ON + + + ( + + + + expr_list + + + + ) + + + + +

referenced by: +

+


all_or_distinct:

+ + + + + + + ALL + + + DISTINCT + + + + +

referenced by: +

+


for_locking_item:

+ + + + + + + + for_locking_strength + + + + + opt_locked_rels + + + + + opt_nowait_or_skip + + + + + +

referenced by: +

+


var_list:

+ + + + + + + + var_value + + + + , + + + + +

referenced by: +

+


opt_ordinality:

+ + + + + + + WITH + + + ORDINALITY + + + + +

referenced by: +

+


opt_alias_clause:

+ + + + + + + + alias_clause + + + + + +

referenced by: +

+


joined_table:

+ + + + + + + ( + + + + joined_table + + + + ) + + + + table_ref + + + + CROSS + + + + opt_join_hint + + + + NATURAL + + + + join_type + + + + + opt_join_hint + + + + JOIN + + + + table_ref + + + + + join_type + + + + + opt_join_hint + + + + JOIN + + + + table_ref + + + + + join_qual + + + + + +

referenced by: +

+


alias_clause:

+ + + + + + + AS + + + + table_alias_name + + + + + opt_column_list + + + + + +

referenced by: +

+


func_table:

+ + + + + + + + func_expr_windowless + + + + ROWS + + + FROM + + + ( + + + + rowsfrom_list + + + + ) + + + + +

referenced by: +

+


row_source_extension_stmt:

+ + + + + + + + delete_stmt + + + + + explain_stmt + + + + + insert_stmt + + + + + select_stmt + + + + + show_stmt + + + + + update_stmt + + + + + upsert_stmt + + + + + +

referenced by: +

+


user_priority:

+ + + + + + + LOW + + + NORMAL + + + HIGH + + + + +

referenced by: +

+


alter_table_cmd:

+ + + + + + + RENAME + + + + opt_column + + + + CONSTRAINT + + + + column_name + + + + TO + + + + column_name + + + + ADD + + + COLUMN + + + IF + + + NOT + + + EXISTS + + + + column_def + + + + + table_constraint + + + + + opt_validate_behavior + + + + ALTER + + + + opt_column + + + + + column_name + + + + + alter_column_default + + + + DROP + + + NOT + + + NULL + + + STORED + + + SET + + + NOT + + + NULL + + + + opt_set_data + + + + TYPE + + + + typename + + + + + opt_collate + + + + + opt_alter_column_using + + + + PRIMARY + + + KEY + + + USING + + + COLUMNS + + + ( + + + + index_params + + + + ) + + + + opt_hash_sharded + + + + + opt_interleave + + + + DROP + + + + opt_column + + + + IF + + + EXISTS + + + + column_name + + + + CONSTRAINT + + + IF + + + EXISTS + + + + constraint_name + + + + + opt_drop_behavior + + + + VALIDATE + + + CONSTRAINT + + + + constraint_name + + + + EXPERIMENTAL_AUDIT + + + SET + + + + audit_mode + + + + + partition_by + + + + + +

referenced by: +

+


var_set_list:

+ + + + + + + + var_name + + + + = + + + COPY + + + FROM + + + PARENT + + + + var_value + + + + , + + + + var_name + + + + = + + + + var_value + + + + COPY + + + FROM + + + PARENT + + + + +

referenced by: +

+


alter_index_cmd:

+ + + + + + + + partition_by + + + + + +

referenced by: +

+


sequence_option_elem:

+ + + + + + + NO + + + CYCLE + + + MINVALUE + + + MAXVALUE + + + OWNED + + + BY + + + NONE + + + + column_path + + + + INCREMENT + + + BY + + + MINVALUE + + + MAXVALUE + + + START + + + WITH + + + + signed_iconst64 + + + + VIRTUAL + + + + +

referenced by: +

+


password_clause:

+ + + + + + + PASSWORD + + + + string_or_placeholder + + + + NULL + + + + +

referenced by: +

+


valid_until_clause:

+ + + + + + + VALID + + + UNTIL + + + + string_or_placeholder + + + + NULL + + + + +

referenced by: +

+


opt_asc_desc:

+ + + + + + + ASC + + + DESC + + + + +

referenced by: +

+


opt_nulls_order:

+ + + + + + + NULLS + + + FIRST + + + LAST + + + + +

referenced by: +

+


list_partitions:

+ + + + + + + + list_partition + + + + , + + + + +

referenced by: +

+


range_partitions:

+ + + + + + + + range_partition + + + + , + + + + +

referenced by: +

+


create_as_col_qual_list:

+ + + + + + + + create_as_col_qualification + + + + + +

referenced by: +

+


create_as_constraint_def:

+ + + + + + + + create_as_constraint_elem + + + + + +

referenced by: +

+


index_flags_param:

+ + + + + + + FORCE_INDEX + + + = + + + + index_name + + + + NO_INDEX_JOIN + + + + +

referenced by: +

+


col_qualification:

+ + + + + + + CONSTRAINT + + + + constraint_name + + + + + col_qualification_elem + + + + COLLATE + + + + collation_name + + + + FAMILY + + + + family_name + + + + CREATE + + + FAMILY + + + + family_name + + + + IF + + + NOT + + + EXISTS + + + FAMILY + + + + family_name + + + + + +

referenced by: +

+


key_match:

+ + + + + + + MATCH + + + SIMPLE + + + FULL + + + + +

referenced by: +

+


reference_actions:

+ + + + + + + + reference_on_update + + + + + reference_on_delete + + + + + reference_on_delete + + + + + reference_on_update + + + + + +

referenced by: +

+


numeric:

+ + + + + + + INT + + + INTEGER + + + INT2 + + + SMALLINT + + + INT4 + + + INT8 + + + INT64 + + + BIGINT + + + REAL + + + FLOAT4 + + + FLOAT8 + + + FLOAT + + + + opt_float + + + + DOUBLE + + + PRECISION + + + DECIMAL + + + DEC + + + NUMERIC + + + + opt_numeric_modifiers + + + + BOOLEAN + + + BOOL + + + + +

referenced by: +

+


bit_without_length:

+ + + + + + + BIT + + + VARYING + + + VARBIT + + + + +

referenced by: +

+


character_without_length:

+ + + + + + + + character_base + + + + + +

referenced by: +

+


const_datetime:

+ + + + + + + DATE + + + TIME + + + TIMESTAMP + + + ( + + + ICONST + + + ) + + + + opt_timezone + + + + TIMETZ + + + TIMESTAMPTZ + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


const_json:

+ + + + + + + JSON + + + JSONB + + + + +

referenced by: +

+


opt_interval_qualifier:

+ + + + + + + + interval_qualifier + + + + + +

referenced by: +

+


func_application:

+ + + + + + + + func_name + + + + ( + + + ALL + + + + expr_list + + + + + opt_sort_clause + + + + DISTINCT + + + + expr_list + + + + * + + + ) + + + + +

referenced by: +

+


filter_clause:

+ + + + + + + FILTER + + + ( + + + WHERE + + + + a_expr + + + + ) + + + + +

referenced by: +

+


over_clause:

+ + + + + + + OVER + + + + window_specification + + + + + window_name + + + + + +

referenced by: +

+


func_expr_common_subexpr:

+ + + + + + + COLLATION + + + FOR + + + ( + + + IF + + + ( + + + + a_expr + + + + , + + + NULLIF + + + IFNULL + + + ( + + + IFERROR + + + ( + + + + a_expr + + + + , + + + + a_expr + + + + , + + + ISERROR + + + ( + + + + a_expr + + + + , + + + + a_expr + + + + CAST + + + ( + + + + a_expr + + + + AS + + + + cast_target + + + + ANNOTATE_TYPE + + + ( + + + + a_expr + + + + , + + + + typename + + + + COALESCE + + + ( + + + + expr_list + + + + ) + + + CURRENT_DATE + + + CURRENT_SCHEMA + + + CURRENT_CATALOG + + + CURRENT_TIMESTAMP + + + CURRENT_TIME + + + LOCALTIMESTAMP + + + LOCALTIME + + + CURRENT_USER + + + CURRENT_ROLE + + + SESSION_USER + + + USER + + + + special_function + + + + + +

referenced by: +

+


opt_expr_list:

+ + + + + + + + expr_list + + + + + +

referenced by: +

+


expr_tuple_unambiguous:

+ + + + + + + ( + + + + tuple1_unambiguous_values + + + + ) + + + + +

referenced by: +

+


array_expr_list:

+ + + + + + + + array_expr + + + + , + + + + +

referenced by: +

+


opt_slice_bound:

+ + + + + + + + a_expr + + + + + +

referenced by: +

+


when_clause:

+ + + + + + + WHEN + + + + a_expr + + + + THEN + + + + a_expr + + + + + +

referenced by: +

+


opt_varying:

+ + + + + + + VARYING + + + + +

referenced by: +

+


character_base:

+ + + + + + + + char_aliases + + + + VARYING + + + VARCHAR + + + STRING + + + + +

referenced by: +

+


interval_qualifier:

+ + + + + + + YEAR + + + TO + + + MONTH + + + MONTH + + + DAY + + + TO + + + HOUR + + + MINUTE + + + + interval_second + + + + HOUR + + + TO + + + MINUTE + + + + interval_second + + + + MINUTE + + + TO + + + + interval_second + + + + + interval_second + + + + + +

referenced by: +

+


window_definition_list:

+ + + + + + + + window_definition + + + + , + + + + +

referenced by: +

+


for_locking_strength:

+ + + + + + + FOR + + + NO + + + KEY + + + UPDATE + + + KEY + + + SHARE + + + + +

referenced by: +

+


opt_locked_rels:

+ + + + + + + OF + + + + table_name_list + + + + + +

referenced by: +

+


opt_nowait_or_skip:

+ + + + + + + SKIP + + + LOCKED + + + NOWAIT + + + + +

referenced by: +

+


opt_join_hint:

+ + + + + + + HASH + + + MERGE + + + LOOKUP + + + + +

referenced by: +

+


join_type:

+ + + + + + + FULL + + + LEFT + + + RIGHT + + + + join_outer + + + + INNER + + + + +

referenced by: +

+


join_qual:

+ + + + + + + USING + + + ( + + + + name_list + + + + ) + + + ON + + + + a_expr + + + + + +

referenced by: +

+


func_expr_windowless:

+ + + + + + + + func_application + + + + + func_expr_common_subexpr + + + + + +

referenced by: +

+


rowsfrom_list:

+ + + + + + + + rowsfrom_item + + + + , + + + + +

referenced by: +

+


opt_column:

+ + + + + + + COLUMN + + + + +

referenced by: +

+


alter_column_default:

+ + + + + + + SET + + + DEFAULT + + + + a_expr + + + + DROP + + + DEFAULT + + + + +

referenced by: +

+


opt_set_data:

+ + + + + + + SET + + + DATA + + + + +

referenced by: +

+


opt_collate:

+ + + + + + + COLLATE + + + + collation_name + + + + + +

referenced by: +

+


opt_alter_column_using:

+ + + + + + + USING + + + + a_expr + + + + + +

referenced by: +

+


opt_validate_behavior:

+ + + + + + + NOT + + + VALID + + + + +

referenced by: +

+


audit_mode:

+ + + + + + + READ + + + WRITE + + + OFF + + + + +

referenced by: +

+


signed_iconst64:

+ + + + + + + + signed_iconst + + + + + +

referenced by: +

+


list_partition:

+ + + + + + + + partition + + + + VALUES + + + IN + + + ( + + + + expr_list + + + + ) + + + + opt_partition_by + + + + + +

referenced by: +

+


range_partition:

+ + + + + + + + partition + + + + VALUES + + + FROM + + + ( + + + + expr_list + + + + ) + + + TO + + + ( + + + + expr_list + + + + ) + + + + opt_partition_by + + + + + +

referenced by: +

+


create_as_col_qualification:

+ + + + + + + + create_as_col_qualification_elem + + + + FAMILY + + + + family_name + + + + + +

referenced by: +

+


create_as_constraint_elem:

+ + + + + + + PRIMARY + + + KEY + + + ( + + + + create_as_params + + + + ) + + + + +

referenced by: +

+


col_qualification_elem:

+ + + + + + + NOT + + + NULL + + + UNIQUE + + + PRIMARY + + + KEY + + + USING + + + HASH + + + WITH + + + BUCKET_COUNT + + + = + + + + a_expr + + + + CHECK + + + ( + + + + a_expr + + + + ) + + + DEFAULT + + + + b_expr + + + + REFERENCES + + + + table_name + + + + + opt_name_parens + + + + + key_match + + + + + reference_actions + + + + AS + + + ( + + + + a_expr + + + + ) + + + STORED + + + + +

referenced by: +

+


family_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


reference_on_update:

+ + + + + + + ON + + + UPDATE + + + + reference_action + + + + + +

referenced by: +

+


reference_on_delete:

+ + + + + + + ON + + + DELETE + + + + reference_action + + + + + +

referenced by: +

+


opt_float:

+ + + + + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


opt_numeric_modifiers:

+ + + + + + + ( + + + ICONST + + + , + + + ICONST + + + ) + + + + +

referenced by: +

+


opt_timezone:

+ + + + + + + WITH + + + WITHOUT + + + TIME + + + ZONE + + + + +

referenced by: +

+


func_name:

+ + + + + + + + type_function_name + + + + + prefixed_column_path + + + + + +

referenced by: +

+


window_specification:

+ + + + + + + ( + + + + opt_existing_window_name + + + + + opt_partition_clause + + + + + opt_sort_clause + + + + + opt_frame_clause + + + + ) + + + + +

referenced by: +

+


window_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


special_function:

+ + + + + + + CURRENT_DATE + + + CURRENT_SCHEMA + + + CURRENT_USER + + + ( + + + EXTRACT + + + EXTRACT_DURATION + + + ( + + + + extract_list + + + + OVERLAY + + + ( + + + + overlay_list + + + + POSITION + + + ( + + + + position_list + + + + SUBSTRING + + + ( + + + + substr_list + + + + GREATEST + + + LEAST + + + ( + + + + expr_list + + + + CURRENT_TIMESTAMP + + + CURRENT_TIME + + + LOCALTIMESTAMP + + + LOCALTIME + + + ( + + + + a_expr + + + + TRIM + + + ( + + + BOTH + + + LEADING + + + TRAILING + + + + trim_list + + + + ) + + + + +

referenced by: +

+


tuple1_unambiguous_values:

+ + + + + + + + a_expr + + + + , + + + + expr_list + + + + + +

referenced by: +

+


char_aliases:

+ + + + + + + CHAR + + + CHARACTER + + + + +

referenced by: +

+


interval_second:

+ + + + + + + SECOND + + + ( + + + ICONST + + + ) + + + + +

referenced by: +

+


window_definition:

+ + + + + + + + window_name + + + + AS + + + + window_specification + + + + + +

referenced by: +

+


join_outer:

+ + + + + + + OUTER + + + + +

referenced by: +

+


rowsfrom_item:

+ + + + + + + + func_expr_windowless + + + + + +

referenced by: +

+


signed_iconst:

+ + + + + + + ICONST + + + + only_signed_iconst + + + + + +

referenced by: +

+


create_as_col_qualification_elem:

+ + + + + + + PRIMARY + + + KEY + + + + +

referenced by: +

+


create_as_params:

+ + + + + + + + create_as_param + + + + , + + + + +

referenced by: +

+


opt_name_parens:

+ + + + + + + ( + + + + name + + + + ) + + + + +

referenced by: +

+


reference_action:

+ + + + + + + NO + + + ACTION + + + RESTRICT + + + CASCADE + + + SET + + + NULL + + + DEFAULT + + + + +

referenced by: +

+


type_function_name:

+ + + + + + + identifier + + + + unreserved_keyword + + + + + type_func_name_keyword + + + + + +

referenced by: +

+


opt_existing_window_name:

+ + + + + + + + name + + + + + +

referenced by: +

+


opt_partition_clause:

+ + + + + + + PARTITION + + + BY + + + + expr_list + + + + + +

referenced by: +

+


opt_frame_clause:

+ + + + + + + RANGE + + + ROWS + + + GROUPS + + + + frame_extent + + + + + opt_frame_exclusion + + + + + +

referenced by: +

+


extract_list:

+ + + + + + + + extract_arg + + + + FROM + + + + a_expr + + + + + expr_list + + + + + +

referenced by: +

+


overlay_list:

+ + + + + + + + a_expr + + + + + overlay_placing + + + + + substr_from + + + + + substr_for + + + + + expr_list + + + + + +

referenced by: +

+


position_list:

+ + + + + + + + b_expr + + + + IN + + + + b_expr + + + + + +

referenced by: +

+


substr_list:

+ + + + + + + + a_expr + + + + + substr_from + + + + + substr_for + + + + + substr_for + + + + + substr_from + + + + + opt_expr_list + + + + + +

referenced by: +

+


trim_list:

+ + + + + + + + a_expr + + + + FROM + + + + expr_list + + + + + +

referenced by: +

+


create_as_param:

+ + + + + + + + column_name + + + + + +

referenced by: +

+


frame_extent:

+ + + + + + + BETWEEN + + + + frame_bound + + + + AND + + + + frame_bound + + + + + +

referenced by: +

+


opt_frame_exclusion:

+ + + + + + + EXCLUDE + + + CURRENT + + + ROW + + + GROUP + + + TIES + + + NO + + + OTHERS + + + + +

referenced by: +

+


extract_arg:

+ + + + + + + identifier + + + YEAR + + + MONTH + + + DAY + + + HOUR + + + MINUTE + + + SECOND + + + SCONST + + + + +

referenced by: +

+


overlay_placing:

+ + + + + + + PLACING + + + + a_expr + + + + + +

referenced by: +

+


substr_from:

+ + + + + + + FROM + + + + a_expr + + + + + +

referenced by: +

+


substr_for:

+ + + + + + + FOR + + + + a_expr + + + + + +

referenced by: +

+


frame_bound:

+ + + + + + + UNBOUNDED + + + + a_expr + + + + PRECEDING + + + FOLLOWING + + + CURRENT + + + ROW + + + + +

referenced by: +

+


generated by Railroad Diagram Generator

\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/table.html b/_includes/v20.2/sql/diagrams/table.html new file mode 100644 index 00000000000..e69de29bb2d diff --git a/_includes/v20.2/sql/diagrams/table_clause.html b/_includes/v20.2/sql/diagrams/table_clause.html new file mode 100644 index 00000000000..97691481d76 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/table_clause.html @@ -0,0 +1,15 @@ +
+ + + + +TABLE + + + +table_ref + + + + +
diff --git a/_includes/v20.2/sql/diagrams/table_constraint.html b/_includes/v20.2/sql/diagrams/table_constraint.html new file mode 100644 index 00000000000..dd7c5cf0c11 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/table_constraint.html @@ -0,0 +1,150 @@ +
+ + + + +CONSTRAINT + + + +constraint_name + + + +CHECK + + +( + + + +a_expr + + + +) + + +UNIQUE + + +( + + + +index_params + + + +) + + +COVERING + + +STORING + + +( + + + +name_list + + + +) + + + +opt_interleave + + + + +opt_partition_by + + + +PRIMARY + + +KEY + + +( + + + +index_params + + + +) + + +USING + + +HASH + + +WITH + + +BUCKET_COUNT + + += + + +n_buckets + + + +opt_interleave + + + +FOREIGN + + +KEY + + +( + + + +name_list + + + +) + + +REFERENCES + + + +table_name + + + + +opt_column_list + + + + +key_match + + + + +reference_actions + + + + +
diff --git a/_includes/v20.2/sql/diagrams/table_ref.html b/_includes/v20.2/sql/diagrams/table_ref.html new file mode 100644 index 00000000000..db27a233acc --- /dev/null +++ b/_includes/v20.2/sql/diagrams/table_ref.html @@ -0,0 +1,72 @@ + diff --git a/_includes/v20.2/sql/diagrams/truncate.html b/_includes/v20.2/sql/diagrams/truncate.html new file mode 100644 index 00000000000..06cb91a310c --- /dev/null +++ b/_includes/v20.2/sql/diagrams/truncate.html @@ -0,0 +1,28 @@ +
+ + + + + + TRUNCATE + + + TABLE + + + + table_name + + + + , + + + CASCADE + + + RESTRICT + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/unique_column_level.html b/_includes/v20.2/sql/diagrams/unique_column_level.html new file mode 100644 index 00000000000..c7c178e9351 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/unique_column_level.html @@ -0,0 +1,59 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_name + + + + + column_type + + + + UNIQUE + + + + column_constraints + + + + , + + + + column_def + + + + + table_constraints + + + + ) + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/unique_table_level.html b/_includes/v20.2/sql/diagrams/unique_table_level.html new file mode 100644 index 00000000000..e77a972161a --- /dev/null +++ b/_includes/v20.2/sql/diagrams/unique_table_level.html @@ -0,0 +1,63 @@ +
+ + + + + + CREATE + + + TABLE + + + + table_name + + + + ( + + + + column_def + + + + , + + + CONSTRAINT + + + + name + + + + UNIQUE + + + ( + + + + column_name + + + + , + + + ) + + + + table_constraints + + + + ) + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/unsplit_index_at.html b/_includes/v20.2/sql/diagrams/unsplit_index_at.html new file mode 100644 index 00000000000..f2c1e35f6fd --- /dev/null +++ b/_includes/v20.2/sql/diagrams/unsplit_index_at.html @@ -0,0 +1,30 @@ +
+ + + + +ALTER + + +INDEX + + +table_name + +@ + + +index_name + +UNSPLIT + + +AT + + +select_stmt + +ALL + + +
diff --git a/_includes/v20.2/sql/diagrams/unsplit_table_at.html b/_includes/v20.2/sql/diagrams/unsplit_table_at.html new file mode 100644 index 00000000000..797c0beda03 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/unsplit_table_at.html @@ -0,0 +1,25 @@ +
+ + + + +ALTER + + +TABLE + + +table_name + +UNSPLIT + + +AT + + +select_stmt + +ALL + + +
diff --git a/_includes/v20.2/sql/diagrams/update.html b/_includes/v20.2/sql/diagrams/update.html new file mode 100644 index 00000000000..0b81b98ad22 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/update.html @@ -0,0 +1,97 @@ + diff --git a/_includes/v20.2/sql/diagrams/upsert.html b/_includes/v20.2/sql/diagrams/upsert.html new file mode 100644 index 00000000000..9765d9a3b10 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/upsert.html @@ -0,0 +1,57 @@ +
+ + + + +WITH + + +RECURSIVE + + +common_table_expr + +, + + +UPSERT + + +INTO + + +table_name + +AS + + +table_alias_name + +( + + +column_name + +, + + +) + + +select_stmt + +DEFAULT + + +VALUES + + +RETURNING + + +target_list + +NOTHING + + +
diff --git a/_includes/v20.2/sql/diagrams/validate_constraint.html b/_includes/v20.2/sql/diagrams/validate_constraint.html new file mode 100644 index 00000000000..d470d8dd98f --- /dev/null +++ b/_includes/v20.2/sql/diagrams/validate_constraint.html @@ -0,0 +1,36 @@ +
+ + + + + + ALTER + + + TABLE + + + IF + + + EXISTS + + + + table_name + + + + VALIDATE + + + CONSTRAINT + + + + constraint_name + + + + +
\ No newline at end of file diff --git a/_includes/v20.2/sql/diagrams/values_clause.html b/_includes/v20.2/sql/diagrams/values_clause.html new file mode 100644 index 00000000000..34f78e982b4 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/values_clause.html @@ -0,0 +1,27 @@ +
+ + + + +VALUES + + +( + + + +a_expr + + + +, + + +) + + +, + + + +
diff --git a/_includes/v20.2/sql/diagrams/window_definition.html b/_includes/v20.2/sql/diagrams/window_definition.html new file mode 100644 index 00000000000..a5335af0ee9 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/window_definition.html @@ -0,0 +1,28 @@ + diff --git a/_includes/v20.2/sql/diagrams/with_clause.html b/_includes/v20.2/sql/diagrams/with_clause.html new file mode 100644 index 00000000000..be5dfbffb99 --- /dev/null +++ b/_includes/v20.2/sql/diagrams/with_clause.html @@ -0,0 +1,53 @@ + diff --git a/_includes/v20.2/sql/function-special-forms.md b/_includes/v20.2/sql/function-special-forms.md new file mode 100644 index 00000000000..b9ac987444a --- /dev/null +++ b/_includes/v20.2/sql/function-special-forms.md @@ -0,0 +1,29 @@ +| Special form | Equivalent to | +|-----------------------------------------------------------|---------------------------------------------| +| `AT TIME ZONE` | `timezone()` | +| `CURRENT_CATALOG` | `current_catalog()` | +| `COLLATION FOR` | `pg_collation_for()` | +| `CURRENT_DATE` | `current_date()` | +| `CURRENT_ROLE` | `current_user()` | +| `CURRENT_SCHEMA` | `current_schema()` | +| `CURRENT_TIMESTAMP` | `current_timestamp()` | +| `CURRENT_TIME` | `current_time()` | +| `CURRENT_USER` | `current_user()` | +| `EXTRACT( FROM )` | `extract("", )` | +| `EXTRACT_DURATION( FROM )` | `extract_duration("", )` | +| `OVERLAY( PLACING FROM FOR )` | `overlay(, , , )` | +| `OVERLAY( PLACING FROM )` | `overlay(, , )` | +| `POSITION( IN )` | `strpos(, )` | +| `SESSION_USER` | `current_user()` | +| `SUBSTRING( FOR FROM )` | `substring(, , )` | +| `SUBSTRING( FOR )` | `substring(, 1, )` | +| `SUBSTRING( FROM FOR )` | `substring(, , )` | +| `SUBSTRING( FROM )` | `substring(, )` | +| `TRIM( FROM )` | `btrim(, )` | +| `TRIM(, )` | `btrim(, )` | +| `TRIM(FROM )` | `btrim()` | +| `TRIM(LEADING FROM )` | `ltrim(, )` | +| `TRIM(LEADING FROM )` | `ltrim()` | +| `TRIM(TRAILING FROM )` | `rtrim(, )` | +| `TRIM(TRAILING FROM )` | `rtrim()` | +| `USER` | `current_user()` | diff --git a/_includes/v20.2/sql/functions.md b/_includes/v20.2/sql/functions.md new file mode 100644 index 00000000000..b2c9871a0ef --- /dev/null +++ b/_includes/v20.2/sql/functions.md @@ -0,0 +1,1368 @@ +### Array functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
array_append(array: bool[], elem: bool) → bool[]

Appends elem to array, returning the result.

+
array_append(array: bytes[], elem: bytes) → bytes[]

Appends elem to array, returning the result.

+
array_append(array: date[], elem: date) → date[]

Appends elem to array, returning the result.

+
array_append(array: decimal[], elem: decimal) → decimal[]

Appends elem to array, returning the result.

+
array_append(array: float[], elem: float) → float[]

Appends elem to array, returning the result.

+
array_append(array: inet[], elem: inet) → inet[]

Appends elem to array, returning the result.

+
array_append(array: int[], elem: int) → int[]

Appends elem to array, returning the result.

+
array_append(array: interval[], elem: interval) → interval[]

Appends elem to array, returning the result.

+
array_append(array: string[], elem: string) → string[]

Appends elem to array, returning the result.

+
array_append(array: time[], elem: time) → time[]

Appends elem to array, returning the result.

+
array_append(array: timestamp[], elem: timestamp) → timestamp[]

Appends elem to array, returning the result.

+
array_append(array: timestamptz[], elem: timestamptz) → timestamptz[]

Appends elem to array, returning the result.

+
array_append(array: uuid[], elem: uuid) → uuid[]

Appends elem to array, returning the result.

+
array_append(array: oid[], elem: oid) → oid[]

Appends elem to array, returning the result.

+
array_append(array: timetz[], elem: timetz) → timetz[]

Appends elem to array, returning the result.

+
array_append(array: varbit[], elem: varbit) → varbit[]

Appends elem to array, returning the result.

+
array_cat(left: bool[], right: bool[]) → bool[]

Appends two arrays.

+
array_cat(left: bytes[], right: bytes[]) → bytes[]

Appends two arrays.

+
array_cat(left: date[], right: date[]) → date[]

Appends two arrays.

+
array_cat(left: decimal[], right: decimal[]) → decimal[]

Appends two arrays.

+
array_cat(left: float[], right: float[]) → float[]

Appends two arrays.

+
array_cat(left: inet[], right: inet[]) → inet[]

Appends two arrays.

+
array_cat(left: int[], right: int[]) → int[]

Appends two arrays.

+
array_cat(left: interval[], right: interval[]) → interval[]

Appends two arrays.

+
array_cat(left: string[], right: string[]) → string[]

Appends two arrays.

+
array_cat(left: time[], right: time[]) → time[]

Appends two arrays.

+
array_cat(left: timestamp[], right: timestamp[]) → timestamp[]

Appends two arrays.

+
array_cat(left: timestamptz[], right: timestamptz[]) → timestamptz[]

Appends two arrays.

+
array_cat(left: uuid[], right: uuid[]) → uuid[]

Appends two arrays.

+
array_cat(left: oid[], right: oid[]) → oid[]

Appends two arrays.

+
array_cat(left: timetz[], right: timetz[]) → timetz[]

Appends two arrays.

+
array_cat(left: varbit[], right: varbit[]) → varbit[]

Appends two arrays.

+
array_length(input: anyelement[], array_dimension: int) → int

Calculates the length of input on the provided array_dimension. However, because CockroachDB doesn’t yet support multi-dimensional arrays, the only supported array_dimension is 1.

+
array_lower(input: anyelement[], array_dimension: int) → int

Calculates the minimum value of input on the provided array_dimension. However, because CockroachDB doesn’t yet support multi-dimensional arrays, the only supported array_dimension is 1.

+
array_position(array: bool[], elem: bool) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: bytes[], elem: bytes) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: date[], elem: date) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: decimal[], elem: decimal) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: float[], elem: float) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: inet[], elem: inet) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: int[], elem: int) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: interval[], elem: interval) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: string[], elem: string) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: time[], elem: time) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: timestamp[], elem: timestamp) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: timestamptz[], elem: timestamptz) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: uuid[], elem: uuid) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: oid[], elem: oid) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: timetz[], elem: timetz) → int

Return the index of the first occurrence of elem in array.

+
array_position(array: varbit[], elem: varbit) → int

Return the index of the first occurrence of elem in array.

+
array_positions(array: bool[], elem: bool) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: bytes[], elem: bytes) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: date[], elem: date) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: decimal[], elem: decimal) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: float[], elem: float) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: inet[], elem: inet) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: int[], elem: int) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: interval[], elem: interval) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: string[], elem: string) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: time[], elem: time) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: timestamp[], elem: timestamp) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: timestamptz[], elem: timestamptz) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: uuid[], elem: uuid) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: oid[], elem: oid) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: timetz[], elem: timetz) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_positions(array: varbit[], elem: varbit) → int[]

Returns and array of indexes of all occurrences of elem in array.

+
array_prepend(elem: bool, array: bool[]) → bool[]

Prepends elem to array, returning the result.

+
array_prepend(elem: bytes, array: bytes[]) → bytes[]

Prepends elem to array, returning the result.

+
array_prepend(elem: date, array: date[]) → date[]

Prepends elem to array, returning the result.

+
array_prepend(elem: decimal, array: decimal[]) → decimal[]

Prepends elem to array, returning the result.

+
array_prepend(elem: float, array: float[]) → float[]

Prepends elem to array, returning the result.

+
array_prepend(elem: inet, array: inet[]) → inet[]

Prepends elem to array, returning the result.

+
array_prepend(elem: int, array: int[]) → int[]

Prepends elem to array, returning the result.

+
array_prepend(elem: interval, array: interval[]) → interval[]

Prepends elem to array, returning the result.

+
array_prepend(elem: string, array: string[]) → string[]

Prepends elem to array, returning the result.

+
array_prepend(elem: time, array: time[]) → time[]

Prepends elem to array, returning the result.

+
array_prepend(elem: timestamp, array: timestamp[]) → timestamp[]

Prepends elem to array, returning the result.

+
array_prepend(elem: timestamptz, array: timestamptz[]) → timestamptz[]

Prepends elem to array, returning the result.

+
array_prepend(elem: uuid, array: uuid[]) → uuid[]

Prepends elem to array, returning the result.

+
array_prepend(elem: oid, array: oid[]) → oid[]

Prepends elem to array, returning the result.

+
array_prepend(elem: timetz, array: timetz[]) → timetz[]

Prepends elem to array, returning the result.

+
array_prepend(elem: varbit, array: varbit[]) → varbit[]

Prepends elem to array, returning the result.

+
array_remove(array: bool[], elem: bool) → bool[]

Remove from array all elements equal to elem.

+
array_remove(array: bytes[], elem: bytes) → bytes[]

Remove from array all elements equal to elem.

+
array_remove(array: date[], elem: date) → date[]

Remove from array all elements equal to elem.

+
array_remove(array: decimal[], elem: decimal) → decimal[]

Remove from array all elements equal to elem.

+
array_remove(array: float[], elem: float) → float[]

Remove from array all elements equal to elem.

+
array_remove(array: inet[], elem: inet) → inet[]

Remove from array all elements equal to elem.

+
array_remove(array: int[], elem: int) → int[]

Remove from array all elements equal to elem.

+
array_remove(array: interval[], elem: interval) → interval[]

Remove from array all elements equal to elem.

+
array_remove(array: string[], elem: string) → string[]

Remove from array all elements equal to elem.

+
array_remove(array: time[], elem: time) → time[]

Remove from array all elements equal to elem.

+
array_remove(array: timestamp[], elem: timestamp) → timestamp[]

Remove from array all elements equal to elem.

+
array_remove(array: timestamptz[], elem: timestamptz) → timestamptz[]

Remove from array all elements equal to elem.

+
array_remove(array: uuid[], elem: uuid) → uuid[]

Remove from array all elements equal to elem.

+
array_remove(array: oid[], elem: oid) → oid[]

Remove from array all elements equal to elem.

+
array_remove(array: timetz[], elem: timetz) → timetz[]

Remove from array all elements equal to elem.

+
array_remove(array: varbit[], elem: varbit) → varbit[]

Remove from array all elements equal to elem.

+
array_replace(array: bool[], toreplace: bool, replacewith: bool) → bool[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: bytes[], toreplace: bytes, replacewith: bytes) → bytes[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: date[], toreplace: date, replacewith: date) → date[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: decimal[], toreplace: decimal, replacewith: decimal) → decimal[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: float[], toreplace: float, replacewith: float) → float[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: inet[], toreplace: inet, replacewith: inet) → inet[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: int[], toreplace: int, replacewith: int) → int[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: interval[], toreplace: interval, replacewith: interval) → interval[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: string[], toreplace: string, replacewith: string) → string[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: time[], toreplace: time, replacewith: time) → time[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: timestamp[], toreplace: timestamp, replacewith: timestamp) → timestamp[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: timestamptz[], toreplace: timestamptz, replacewith: timestamptz) → timestamptz[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: uuid[], toreplace: uuid, replacewith: uuid) → uuid[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: oid[], toreplace: oid, replacewith: oid) → oid[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: timetz[], toreplace: timetz, replacewith: timetz) → timetz[]

Replace all occurrences of toreplace in array with replacewith.

+
array_replace(array: varbit[], toreplace: varbit, replacewith: varbit) → varbit[]

Replace all occurrences of toreplace in array with replacewith.

+
array_to_string(input: anyelement[], delim: string) → string

Join an array into a string with a delimiter.

+
array_to_string(input: anyelement[], delimiter: string, null: string) → string

Join an array into a string with a delimiter, replacing NULLs with a null string.

+
array_upper(input: anyelement[], array_dimension: int) → int

Calculates the maximum value of input on the provided array_dimension. However, because CockroachDB doesn’t yet support multi-dimensional arrays, the only supported array_dimension is 1.

+
string_to_array(str: string, delimiter: string) → string[]

Split a string into components on a delimiter.

+
string_to_array(str: string, delimiter: string, null: string) → string[]

Split a string into components on a delimiter with a specified string to consider NULL.

+
+ +### BOOL functions + + + + + + + + + + + + + +
Function → ReturnsDescription
ilike_escape(unescaped: string, pattern: string, escape: string) → bool

Matches case insensetively unescaped with pattern using ‘escape’ as an escape token.

+
inet_contained_by_or_equals(val: inet, container: inet) → bool

Test for subnet inclusion or equality, using only the network parts of the addresses. The host part of the addresses is ignored.

+
inet_contains_or_equals(container: inet, val: inet) → bool

Test for subnet inclusion or equality, using only the network parts of the addresses. The host part of the addresses is ignored.

+
inet_same_family(val: inet, val: inet) → bool

Checks if two IP addresses are of the same IP family.

+
like_escape(unescaped: string, pattern: string, escape: string) → bool

Matches unescaped with pattern using ‘escape’ as an escape token.

+
not_ilike_escape(unescaped: string, pattern: string, escape: string) → bool

Checks whether unescaped not matches case insensetively with pattern using ‘escape’ as an escape token.

+
not_like_escape(unescaped: string, pattern: string, escape: string) → bool

Checks whether unescaped not matches with pattern using ‘escape’ as an escape token.

+
not_similar_to_escape(unescaped: string, pattern: string, escape: string) → bool

Checks whether unescaped not matches with pattern using ‘escape’ as an escape token.

+
similar_to_escape(unescaped: string, pattern: string, escape: string) → bool

Matches unescaped with pattern using ‘escape’ as an escape token.

+
+ +### Comparison functions + + + + + + +
Function → ReturnsDescription
greatest(anyelement...) → anyelement

Returns the element with the greatest value.

+
least(anyelement...) → anyelement

Returns the element with the lowest value.

+
+ +### Date and time functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
age(end: timestamptz, begin: timestamptz) → interval

Calculates the interval between begin and end.

+
age(val: timestamptz) → interval

Calculates the interval between val and the current time.

+
clock_timestamp() → timestamp

Returns the current system time on one of the cluster nodes.

+
clock_timestamp() → timestamptz

Returns the current system time on one of the cluster nodes.

+
current_date() → date

Returns the date of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
current_timestamp() → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
current_timestamp() → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
current_timestamp() → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
current_timestamp(precision: int) → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
current_timestamp(precision: int) → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
current_timestamp(precision: int) → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
date_trunc(element: string, input: date) → timestamptz

Truncates input to precision element. Sets all fields that are less +significant than element to zero (or one, for day and month)

+

Compatible elements: millennium, century, decade, year, quarter, month, +week, day, hour, minute, second, millisecond, microsecond.

+
date_trunc(element: string, input: time) → interval

Truncates input to precision element. Sets all fields that are less +significant than element to zero.

+

Compatible elements: hour, minute, second, millisecond, microsecond.

+
date_trunc(element: string, input: timestamp) → timestamp

Truncates input to precision element. Sets all fields that are less +significant than element to zero (or one, for day and month)

+

Compatible elements: millennium, century, decade, year, quarter, month, +week, day, hour, minute, second, millisecond, microsecond.

+
date_trunc(element: string, input: timestamptz) → timestamptz

Truncates input to precision element. Sets all fields that are less +significant than element to zero (or one, for day and month)

+

Compatible elements: millennium, century, decade, year, quarter, month, +week, day, hour, minute, second, millisecond, microsecond.

+
experimental_follower_read_timestamp() → timestamptz

Returns a timestamp which is very likely to be safe to perform +against a follower replica.

+

This function is intended to be used with an AS OF SYSTEM TIME clause to perform +historical reads against a time which is recent but sufficiently old for reads +to be performed against the closest replica as opposed to the currently +leaseholder for a given range.

+

Note that this function requires an enterprise license on a CCL distribution to +return without an error.

+
experimental_strftime(input: date, extract_format: string) → string

From input, extracts and formats the time as identified in extract_format using standard strftime notation (though not all formatting is supported).

+
experimental_strftime(input: timestamp, extract_format: string) → string

From input, extracts and formats the time as identified in extract_format using standard strftime notation (though not all formatting is supported).

+
experimental_strftime(input: timestamptz, extract_format: string) → string

From input, extracts and formats the time as identified in extract_format using standard strftime notation (though not all formatting is supported).

+
experimental_strptime(input: string, format: string) → timestamptz

Returns input as a timestamptz using format (which uses standard strptime formatting).

+
extract(element: string, input: date) → float

Extracts element from input.

+

Compatible elements: millennium, century, decade, year, isoyear, +quarter, month, week, dayofweek, isodow, dayofyear, julian, +hour, minute, second, millisecond, microsecond, epoch

+
extract(element: string, input: interval) → float

Extracts element from input.

+

Compatible elements: millennium, century, decade, year, +month, day, hour, minute, second, millisecond, microsecond, epoch

+
extract(element: string, input: time) → float

Extracts element from input.

+

Compatible elements: hour, minute, second, millisecond, microsecond, epoch

+
extract(element: string, input: timestamp) → float

Extracts element from input.

+

Compatible elements: millennium, century, decade, year, isoyear, +quarter, month, week, dayofweek, isodow, dayofyear, julian, +hour, minute, second, millisecond, microsecond, epoch

+
extract(element: string, input: timestamptz) → float

Extracts element from input.

+

Compatible elements: millennium, century, decade, year, isoyear, +quarter, month, week, dayofweek, isodow, dayofyear, julian, +hour, minute, second, millisecond, microsecond, epoch, +timezone, timezone_hour, timezone_minute

+
extract(element: string, input: timetz) → float

Extracts element from input.

+

Compatible elements: hour, minute, second, millisecond, microsecond, epoch, +timezone, timezone_hour, timezone_minute

+
extract_duration(element: string, input: interval) → int

Extracts element from input. +Compatible elements: hour, minute, second, millisecond, microsecond. +This is deprecated in favor of extract which supports duration.

+
localtimestamp() → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
localtimestamp() → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
localtimestamp() → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
localtimestamp(precision: int) → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
localtimestamp(precision: int) → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
localtimestamp(precision: int) → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
now() → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
now() → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
now() → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
statement_timestamp() → timestamp

Returns the start time of the current statement.

+
statement_timestamp() → timestamptz

Returns the start time of the current statement.

+
timeofday() → string

Returns the current system time on one of the cluster nodes as a string.

+
timezone(timezone: string, time: time) → timetz

Treat given time without time zone as located in the specified time zone.

+
timezone(timezone: string, timestamp: timestamp) → timestamptz

Treat given time stamp without time zone as located in the specified time zone.

+
timezone(timezone: string, timestamptz: timestamptz) → timestamp

Convert given time stamp with time zone to the new time zone, with no time zone designation.

+
timezone(timezone: string, timestamptz_string: string) → timestamp

Convert given time stamp with time zone to the new time zone, with no time zone designation.

+
timezone(timezone: string, timetz: timetz) → timetz

Convert given time with time zone to the new time zone.

+
transaction_timestamp() → date

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
transaction_timestamp() → timestamp

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+
transaction_timestamp() → timestamptz

Returns the time of the current transaction.

+

The value is based on a timestamp picked when the transaction starts +and which stays constant throughout the transaction. This timestamp +has no relationship with the commit order of concurrent transactions.

+

This function is the preferred overload and will be evaluated by default.

+
+ +### ID generation functions + + + + + + + + +
Function → ReturnsDescription
experimental_uuid_v4() → bytes

Returns a UUID.

+
gen_random_uuid() → uuid

Generates a random UUID and returns it as a value of UUID type.

+
unique_rowid() → int

Returns a unique ID used by CockroachDB to generate unique row IDs if a Primary Key isn’t defined for the table. The value is a combination of the insert timestamp and the ID of the node executing the statement, which guarantees this combination is globally unique. However, there can be gaps and the order is not completely guaranteed.

+
uuid_v4() → bytes

Returns a UUID.

+
+ +### INET functions + + + + + + + + + + + + + +
Function → ReturnsDescription
abbrev(val: inet) → string

Converts the combined IP address and prefix length to an abbreviated display format as text.For INET types, this will omit the prefix length if it’s not the default (32 or IPv4, 128 for IPv6)

+

For example, abbrev('192.168.1.2/24') returns '192.168.1.2/24'

+
broadcast(val: inet) → inet

Gets the broadcast address for the network address represented by the value.

+

For example, broadcast('192.168.1.2/24') returns '192.168.1.255/24'

+
family(val: inet) → int

Extracts the IP family of the value; 4 for IPv4, 6 for IPv6.

+

For example, family('::1') returns 6

+
host(val: inet) → string

Extracts the address part of the combined address/prefixlen value as text.

+

For example, host('192.168.1.2/16') returns '192.168.1.2'

+
hostmask(val: inet) → inet

Creates an IP host mask corresponding to the prefix length in the value.

+

For example, hostmask('192.168.1.2/16') returns '0.0.255.255'

+
masklen(val: inet) → int

Retrieves the prefix length stored in the value.

+

For example, masklen('192.168.1.2/16') returns 16

+
netmask(val: inet) → inet

Creates an IP network mask corresponding to the prefix length in the value.

+

For example, netmask('192.168.1.2/16') returns '255.255.0.0'

+
set_masklen(val: inet, prefixlen: int) → inet

Sets the prefix length of val to prefixlen.

+

For example, set_masklen('192.168.1.2', 16) returns '192.168.1.2/16'.

+
text(val: inet) → string

Converts the IP address and prefix length to text.

+
+ +### JSONB functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
array_to_json(array: anyelement[]) → jsonb

Returns the array as JSON or JSONB.

+
array_to_json(array: anyelement[], pretty_bool: bool) → jsonb

Returns the array as JSON or JSONB.

+
json_array_length(json: jsonb) → int

Returns the number of elements in the outermost JSON or JSONB array.

+
json_build_array(anyelement...) → jsonb

Builds a possibly-heterogeneously-typed JSON or JSONB array out of a variadic argument list.

+
json_build_object(anyelement...) → jsonb

Builds a JSON object out of a variadic argument list.

+
json_extract_path(jsonb, string...) → jsonb

Returns the JSON value pointed to by the variadic arguments.

+
json_object(keys: string[], values: string[]) → jsonb

This form of json_object takes keys and values pairwise from two separate arrays. In all other respects it is identical to the one-argument form.

+
json_object(texts: string[]) → jsonb

Builds a JSON or JSONB object out of a text array. The array must have exactly one dimension with an even number of members, in which case they are taken as alternating key/value pairs.

+
json_remove_path(val: jsonb, path: string[]) → jsonb

Remove the specified path from the JSON object.

+
json_set(val: jsonb, path: string[], to: jsonb) → jsonb

Returns the JSON value pointed to by the variadic arguments.

+
json_set(val: jsonb, path: string[], to: jsonb, create_missing: bool) → jsonb

Returns the JSON value pointed to by the variadic arguments. If create_missing is false, new keys will not be inserted to objects and values will not be prepended or appended to arrays.

+
json_strip_nulls(from_json: jsonb) → jsonb

Returns from_json with all object fields that have null values omitted. Other null values are untouched.

+
json_typeof(val: jsonb) → string

Returns the type of the outermost JSON value as a text string.

+
jsonb_array_length(json: jsonb) → int

Returns the number of elements in the outermost JSON or JSONB array.

+
jsonb_build_array(anyelement...) → jsonb

Builds a possibly-heterogeneously-typed JSON or JSONB array out of a variadic argument list.

+
jsonb_build_object(anyelement...) → jsonb

Builds a JSON object out of a variadic argument list.

+
jsonb_extract_path(jsonb, string...) → jsonb

Returns the JSON value pointed to by the variadic arguments.

+
jsonb_insert(target: jsonb, path: string[], new_val: jsonb) → jsonb

Returns the JSON value pointed to by the variadic arguments. new_val will be inserted before path target.

+
jsonb_insert(target: jsonb, path: string[], new_val: jsonb, insert_after: bool) → jsonb

Returns the JSON value pointed to by the variadic arguments. If insert_after is true (default is false), new_val will be inserted after path target.

+
jsonb_object(keys: string[], values: string[]) → jsonb

This form of json_object takes keys and values pairwise from two separate arrays. In all other respects it is identical to the one-argument form.

+
jsonb_object(texts: string[]) → jsonb

Builds a JSON or JSONB object out of a text array. The array must have exactly one dimension with an even number of members, in which case they are taken as alternating key/value pairs.

+
jsonb_pretty(val: jsonb) → string

Returns the given JSON value as a STRING indented and with newlines.

+
jsonb_set(val: jsonb, path: string[], to: jsonb) → jsonb

Returns the JSON value pointed to by the variadic arguments.

+
jsonb_set(val: jsonb, path: string[], to: jsonb, create_missing: bool) → jsonb

Returns the JSON value pointed to by the variadic arguments. If create_missing is false, new keys will not be inserted to objects and values will not be prepended or appended to arrays.

+
jsonb_strip_nulls(from_json: jsonb) → jsonb

Returns from_json with all object fields that have null values omitted. Other null values are untouched.

+
jsonb_typeof(val: jsonb) → string

Returns the type of the outermost JSON value as a text string.

+
to_json(val: anyelement) → jsonb

Returns the value as JSON or JSONB.

+
to_jsonb(val: anyelement) → jsonb

Returns the value as JSON or JSONB.

+
+ +### Math and numeric functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
abs(val: decimal) → decimal

Calculates the absolute value of val.

+
abs(val: float) → float

Calculates the absolute value of val.

+
abs(val: int) → int

Calculates the absolute value of val.

+
acos(val: float) → float

Calculates the inverse cosine of val.

+
asin(val: float) → float

Calculates the inverse sine of val.

+
atan(val: float) → float

Calculates the inverse tangent of val.

+
atan2(x: float, y: float) → float

Calculates the inverse tangent of x/y.

+
cbrt(val: decimal) → decimal

Calculates the cube root (∛) of val.

+
cbrt(val: float) → float

Calculates the cube root (∛) of val.

+
ceil(val: decimal) → decimal

Calculates the smallest integer not smaller than val.

+
ceil(val: float) → float

Calculates the smallest integer not smaller than val.

+
ceil(val: int) → float

Calculates the smallest integer not smaller than val.

+
ceiling(val: decimal) → decimal

Calculates the smallest integer not smaller than val.

+
ceiling(val: float) → float

Calculates the smallest integer not smaller than val.

+
ceiling(val: int) → float

Calculates the smallest integer not smaller than val.

+
cos(val: float) → float

Calculates the cosine of val.

+
cot(val: float) → float

Calculates the cotangent of val.

+
crc32c(bytes...) → int

Calculates the CRC-32 hash using the Castagnoli polynomial.

+
crc32c(string...) → int

Calculates the CRC-32 hash using the Castagnoli polynomial.

+
crc32ieee(bytes...) → int

Calculates the CRC-32 hash using the IEEE polynomial.

+
crc32ieee(string...) → int

Calculates the CRC-32 hash using the IEEE polynomial.

+
degrees(val: float) → float

Converts val as a radian value to a degree value.

+
div(x: decimal, y: decimal) → decimal

Calculates the integer quotient of x/y.

+
div(x: float, y: float) → float

Calculates the integer quotient of x/y.

+
div(x: int, y: int) → int

Calculates the integer quotient of x/y.

+
exp(val: decimal) → decimal

Calculates e ^ val.

+
exp(val: float) → float

Calculates e ^ val.

+
floor(val: decimal) → decimal

Calculates the largest integer not greater than val.

+
floor(val: float) → float

Calculates the largest integer not greater than val.

+
floor(val: int) → float

Calculates the largest integer not greater than val.

+
fnv32(bytes...) → int

Calculates the 32-bit FNV-1 hash value of a set of values.

+
fnv32(string...) → int

Calculates the 32-bit FNV-1 hash value of a set of values.

+
fnv32a(bytes...) → int

Calculates the 32-bit FNV-1a hash value of a set of values.

+
fnv32a(string...) → int

Calculates the 32-bit FNV-1a hash value of a set of values.

+
fnv64(bytes...) → int

Calculates the 64-bit FNV-1 hash value of a set of values.

+
fnv64(string...) → int

Calculates the 64-bit FNV-1 hash value of a set of values.

+
fnv64a(bytes...) → int

Calculates the 64-bit FNV-1a hash value of a set of values.

+
fnv64a(string...) → int

Calculates the 64-bit FNV-1a hash value of a set of values.

+
isnan(val: decimal) → bool

Returns true if val is NaN, false otherwise.

+
isnan(val: float) → bool

Returns true if val is NaN, false otherwise.

+
ln(val: decimal) → decimal

Calculates the natural log of val.

+
ln(val: float) → float

Calculates the natural log of val.

+
log(b: decimal, x: decimal) → decimal

Calculates the base b log of val.

+
log(b: float, x: float) → float

Calculates the base b log of val.

+
log(val: decimal) → decimal

Calculates the base 10 log of val.

+
log(val: float) → float

Calculates the base 10 log of val.

+
mod(x: decimal, y: decimal) → decimal

Calculates x%y.

+
mod(x: float, y: float) → float

Calculates x%y.

+
mod(x: int, y: int) → int

Calculates x%y.

+
pi() → float

Returns the value for pi (3.141592653589793).

+
pow(x: decimal, y: decimal) → decimal

Calculates x^y.

+
pow(x: float, y: float) → float

Calculates x^y.

+
pow(x: int, y: int) → int

Calculates x^y.

+
power(x: decimal, y: decimal) → decimal

Calculates x^y.

+
power(x: float, y: float) → float

Calculates x^y.

+
power(x: int, y: int) → int

Calculates x^y.

+
radians(val: float) → float

Converts val as a degree value to a radians value.

+
random() → float

Returns a random float between 0 and 1.

+
round(input: decimal, decimal_accuracy: int) → decimal

Keeps decimal_accuracy number of figures to the right of the zero position in input using half away from zero rounding. If decimal_accuracy is not in the range -2^31…(2^31-1), the results are undefined.

+
round(input: float, decimal_accuracy: int) → float

Keeps decimal_accuracy number of figures to the right of the zero position in input using half to even (banker’s) rounding.

+
round(val: decimal) → decimal

Rounds val to the nearest integer, half away from zero: round(+/-2.4) = +/-2, round(+/-2.5) = +/-3.

+
round(val: float) → float

Rounds val to the nearest integer using half to even (banker’s) rounding.

+
sign(val: decimal) → decimal

Determines the sign of val: 1 for positive; 0 for 0 values; -1 for negative.

+
sign(val: float) → float

Determines the sign of val: 1 for positive; 0 for 0 values; -1 for negative.

+
sign(val: int) → int

Determines the sign of val: 1 for positive; 0 for 0 values; -1 for negative.

+
sin(val: float) → float

Calculates the sine of val.

+
sqrt(val: decimal) → decimal

Calculates the square root of val.

+
sqrt(val: float) → float

Calculates the square root of val.

+
tan(val: float) → float

Calculates the tangent of val.

+
trunc(val: decimal) → decimal

Truncates the decimal values of val.

+
trunc(val: float) → float

Truncates the decimal values of val.

+
width_bucket(operand: decimal, b1: decimal, b2: decimal, count: int) → int

return the bucket number to which operand would be assigned in a histogram having count equal-width buckets spanning the range b1 to b2.

+
width_bucket(operand: int, b1: int, b2: int, count: int) → int

return the bucket number to which operand would be assigned in a histogram having count equal-width buckets spanning the range b1 to b2.

+
width_bucket(operand: anyelement, thresholds: anyelement[]) → int

return the bucket number to which operand would be assigned given an array listing the lower bounds of the buckets; returns 0 for an input less than the first lower bound; the thresholds array must be sorted, smallest first, or unexpected results will be obtained

+
+ +### Sequence functions + + + + + + + + + +
Function → ReturnsDescription
currval(sequence_name: string) → int

Returns the latest value obtained with nextval for this sequence in this session.

+
lastval() → int

Return value most recently obtained with nextval in this session.

+
nextval(sequence_name: string) → int

Advances the given sequence and returns its new value.

+
setval(sequence_name: string, value: int) → int

Set the given sequence’s current value. The next call to nextval will return value + Increment

+
setval(sequence_name: string, value: int, is_called: bool) → int

Set the given sequence’s current value. If is_called is false, the next call to nextval will return value; otherwise value + Increment.

+
+ +### Set-returning functions + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
aclexplode(aclitems: string[]) → tuple{oid AS grantor, oid AS grantee, string AS privilege_type, bool AS is_grantable}

Produces a virtual table containing aclitem stuff (returns no rows as this feature is unsupported in CockroachDB)

+
crdb_internal.testing_callback(name: string) → int

For internal CRDB testing only. The function calls a callback identified by name registered with the server by the test.

+
crdb_internal.unary_table() → tuple

Produces a virtual table containing a single row with no values.

+

This function is used only by CockroachDB’s developers for testing purposes.

+
generate_series(start: int, end: int) → int

Produces a virtual table containing the integer values from start to end, inclusive.

+
generate_series(start: int, end: int, step: int) → int

Produces a virtual table containing the integer values from start to end, inclusive, by increment of step.

+
generate_series(start: timestamp, end: timestamp, step: interval) → timestamp

Produces a virtual table containing the timestamp values from start to end, inclusive, by increment of step.

+
generate_subscripts(array: anyelement[]) → int

Returns a series comprising the given array’s subscripts.

+
generate_subscripts(array: anyelement[], dim: int) → int

Returns a series comprising the given array’s subscripts.

+
generate_subscripts(array: anyelement[], dim: int, reverse: bool) → int

Returns a series comprising the given array’s subscripts.

+

When reverse is true, the series is returned in reverse order.

+
information_schema._pg_expandarray(input: anyelement[]) → anyelement

Returns the input array as a set of rows with an index

+
json_array_elements(input: jsonb) → jsonb

Expands a JSON array to a set of JSON values.

+
json_array_elements_text(input: jsonb) → string

Expands a JSON array to a set of text values.

+
json_each(input: jsonb) → tuple{string AS key, jsonb AS value}

Expands the outermost JSON or JSONB object into a set of key/value pairs.

+
json_each_text(input: jsonb) → tuple{string AS key, string AS value}

Expands the outermost JSON or JSONB object into a set of key/value pairs. The returned values will be of type text.

+
json_object_keys(input: jsonb) → string

Returns sorted set of keys in the outermost JSON object.

+
jsonb_array_elements(input: jsonb) → jsonb

Expands a JSON array to a set of JSON values.

+
jsonb_array_elements_text(input: jsonb) → string

Expands a JSON array to a set of text values.

+
jsonb_each(input: jsonb) → tuple{string AS key, jsonb AS value}

Expands the outermost JSON or JSONB object into a set of key/value pairs.

+
jsonb_each_text(input: jsonb) → tuple{string AS key, string AS value}

Expands the outermost JSON or JSONB object into a set of key/value pairs. The returned values will be of type text.

+
jsonb_object_keys(input: jsonb) → string

Returns sorted set of keys in the outermost JSON object.

+
pg_get_keywords() → tuple{string AS word, string AS catcode, string AS catdesc}

Produces a virtual table containing the keywords known to the SQL parser.

+
unnest(anyelement[], anyelement[], anyelement[]...) → tuple

Returns the input arrays as a set of rows

+
unnest(input: anyelement[]) → anyelement

Returns the input array as a set of rows

+
+ +### String and byte functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
ascii(val: string) → int

Returns the character code of the first character in val. Despite the name, the function supports Unicode too.

+
bit_length(val: bytes) → int

Calculates the number of bits used to represent val.

+
bit_length(val: string) → int

Calculates the number of bits used to represent val.

+
bit_length(val: varbit) → int

Calculates the number of bits used to represent val.

+
btrim(input: string, trim_chars: string) → string

Removes any characters included in trim_chars from the beginning or end of input (applies recursively).

+

For example, btrim('doggie', 'eod') returns ggi.

+
btrim(val: string) → string

Removes all spaces from the beginning and end of val.

+
char_length(val: bytes) → int

Calculates the number of bytes in val.

+
char_length(val: string) → int

Calculates the number of characters in val.

+
character_length(val: bytes) → int

Calculates the number of bytes in val.

+
character_length(val: string) → int

Calculates the number of characters in val.

+
chr(val: int) → string

Returns the character with the code given in val. Inverse function of ascii().

+
concat(string...) → string

Concatenates a comma-separated list of strings.

+
concat_ws(string...) → string

Uses the first argument as a separator between the concatenation of the subsequent arguments.

+

For example concat_ws('!','wow','great') returns wow!great.

+
convert_from(str: bytes, enc: string) → string

Decode the bytes in str into a string using encoding enc. Supports encodings ‘UTF8’ and ‘LATIN1’.

+
convert_to(str: string, enc: string) → bytes

Encode the string str as a byte array using encoding enc. Supports encodings ‘UTF8’ and ‘LATIN1’.

+
decode(text: string, format: string) → bytes

Decodes data using format (hex / escape / base64).

+
encode(data: bytes, format: string) → string

Encodes data using format (hex / escape / base64).

+
from_ip(val: bytes) → string

Converts the byte string representation of an IP to its character string representation.

+
from_uuid(val: bytes) → string

Converts the byte string representation of a UUID to its character string representation.

+
get_bit(bit_string: varbit, index: int) → int

Extracts a bit at given index in the bit array.

+
get_bit(byte_string: bytes, index: int) → int

Extracts a bit at given index in the byte array.

+
initcap(val: string) → string

Capitalizes the first letter of val.

+
left(input: bytes, return_set: int) → bytes

Returns the first return_set bytes from input.

+
left(input: string, return_set: int) → string

Returns the first return_set characters from input.

+
length(val: bytes) → int

Calculates the number of bytes in val.

+
length(val: string) → int

Calculates the number of characters in val.

+
length(val: varbit) → int

Calculates the number of bits in val.

+
lower(val: string) → string

Converts all characters in val to their lower-case equivalents.

+
lpad(string: string, length: int) → string

Pads string to length by adding ’ ’ to the left of string.If string is longer than length it is truncated.

+
lpad(string: string, length: int, fill: string) → string

Pads string by adding fill to the left of string to make it length. If string is longer than length it is truncated.

+
ltrim(input: string, trim_chars: string) → string

Removes any characters included in trim_chars from the beginning (left-hand side) of input (applies recursively).

+

For example, ltrim('doggie', 'od') returns ggie.

+
ltrim(val: string) → string

Removes all spaces from the beginning (left-hand side) of val.

+
md5(bytes...) → string

Calculates the MD5 hash value of a set of values.

+
md5(string...) → string

Calculates the MD5 hash value of a set of values.

+
octet_length(val: bytes) → int

Calculates the number of bytes used to represent val.

+
octet_length(val: string) → int

Calculates the number of bytes used to represent val.

+
octet_length(val: varbit) → int

Calculates the number of bits used to represent val.

+
overlay(input: string, overlay_val: string, start_pos: int) → string

Replaces characters in input with overlay_val starting at start_pos (begins at 1).

+

For example, overlay('doggie', 'CAT', 2) returns dCATie.

+
overlay(input: string, overlay_val: string, start_pos: int, end_pos: int) → string

Deletes the characters in input between start_pos and end_pos (count starts at 1), and then insert overlay_val at start_pos.

+
pg_collation_for(str: anyelement) → string

Returns the collation of the argument

+
quote_ident(val: string) → string

Return val suitably quoted to serve as identifier in a SQL statement.

+
quote_literal(val: string) → string

Return val suitably quoted to serve as string literal in a SQL statement.

+
quote_literal(val: anyelement) → string

Coerce val to a string and then quote it as a literal.

+
quote_nullable(val: string) → string

Coerce val to a string and then quote it as a literal. If val is NULL, returns ‘NULL’.

+
quote_nullable(val: anyelement) → string

Coerce val to a string and then quote it as a literal. If val is NULL, returns ‘NULL’.

+
regexp_extract(input: string, regex: string) → string

Returns the first match for the Regular Expression regex in input.

+
regexp_replace(input: string, regex: string, replace: string) → string

Replaces matches for the Regular Expression regex in input with the Regular Expression replace.

+
regexp_replace(input: string, regex: string, replace: string, flags: string) → string

Replaces matches for the regular expression regex in input with the regular expression replace using flags.

+

CockroachDB supports the following flags:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FlagDescription
cCase-sensitive matching
gGlobal matching (match each substring instead of only the first)
iCase-insensitive matching
m or nNewline-sensitive (see below)
pPartial newline-sensitive matching (see below)
sNewline-insensitive (default)
wInverse partial newline-sensitive matching (see below)
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Mode. and [^...] match newlines^ and $ match line boundaries
syesno
wyesyes
pnono
m/nnoyes
+
repeat(input: string, repeat_counter: int) → string

Concatenates input repeat_counter number of times.

+

For example, repeat('dog', 2) returns dogdog.

+
replace(input: string, find: string, replace: string) → string

Replaces all occurrences of find with replace in input

+
reverse(val: string) → string

Reverses the order of the string’s characters.

+
right(input: bytes, return_set: int) → bytes

Returns the last return_set bytes from input.

+
right(input: string, return_set: int) → string

Returns the last return_set characters from input.

+
rpad(string: string, length: int) → string

Pads string to length by adding ’ ’ to the right of string. If string is longer than length it is truncated.

+
rpad(string: string, length: int, fill: string) → string

Pads string to length by adding fill to the right of string. If string is longer than length it is truncated.

+
rtrim(input: string, trim_chars: string) → string

Removes any characters included in trim_chars from the end (right-hand side) of input (applies recursively).

+

For example, rtrim('doggie', 'ei') returns dogg.

+
rtrim(val: string) → string

Removes all spaces from the end (right-hand side) of val.

+
set_bit(bit_string: varbit, index: int, to_set: int) → varbit

Updates a bit at given index in the bit array.

+
set_bit(byte_string: bytes, index: int, to_set: int) → bytes

Updates a bit at given index in the byte array.

+
sha1(bytes...) → string

Calculates the SHA1 hash value of a set of values.

+
sha1(string...) → string

Calculates the SHA1 hash value of a set of values.

+
sha256(bytes...) → string

Calculates the SHA256 hash value of a set of values.

+
sha256(string...) → string

Calculates the SHA256 hash value of a set of values.

+
sha512(bytes...) → string

Calculates the SHA512 hash value of a set of values.

+
sha512(string...) → string

Calculates the SHA512 hash value of a set of values.

+
split_part(input: string, delimiter: string, return_index_pos: int) → string

Splits input on delimiter and return the value in the return_index_pos position (starting at 1).

+

For example, split_part('123.456.789.0','.',3)returns 789.

+
strpos(input: bytes, find: bytes) → int

Calculates the position where the byte subarray find begins in input.

+
strpos(input: string, find: string) → int

Calculates the position where the string find begins in input.

+

For example, strpos('doggie', 'gie') returns 4.

+
strpos(input: varbit, find: varbit) → int

Calculates the position where the bit subarray find begins in input.

+
substr(input: bytes, start_pos: int) → bytes

Returns a byte subarray of input starting at start_pos (count starts at 1).

+
substr(input: bytes, start_pos: int, length: int) → bytes

Returns a byte subarray of input starting at start_pos (count starts at 1) and including up to length characters.

+
substr(input: string, regex: string) → string

Returns a substring of input that matches the regular expression regex.

+
substr(input: string, regex: string, escape_char: string) → string

Returns a substring of input that matches the regular expression regex using escape_char as your escape character instead of \.

+
substr(input: string, start_pos: int) → string

Returns a substring of input starting at start_pos (count starts at 1).

+
substr(input: string, start_pos: int, length: int) → string

Returns a substring of input starting at start_pos (count starts at 1) and including up to length characters.

+
substr(input: varbit, start_pos: int) → varbit

Returns a bit subarray of input starting at start_pos (count starts at 1).

+
substr(input: varbit, start_pos: int, length: int) → varbit

Returns a bit subarray of input starting at start_pos (count starts at 1) and including up to length characters.

+
substring(input: bytes, start_pos: int) → bytes

Returns a byte subarray of input starting at start_pos (count starts at 1).

+
substring(input: bytes, start_pos: int, length: int) → bytes

Returns a byte subarray of input starting at start_pos (count starts at 1) and including up to length characters.

+
substring(input: string, regex: string) → string

Returns a substring of input that matches the regular expression regex.

+
substring(input: string, regex: string, escape_char: string) → string

Returns a substring of input that matches the regular expression regex using escape_char as your escape character instead of \.

+
substring(input: string, start_pos: int) → string

Returns a substring of input starting at start_pos (count starts at 1).

+
substring(input: string, start_pos: int, length: int) → string

Returns a substring of input starting at start_pos (count starts at 1) and including up to length characters.

+
substring(input: varbit, start_pos: int) → varbit

Returns a bit subarray of input starting at start_pos (count starts at 1).

+
substring(input: varbit, start_pos: int, length: int) → varbit

Returns a bit subarray of input starting at start_pos (count starts at 1) and including up to length characters.

+
to_english(val: int) → string

This function enunciates the value of its argument using English cardinals.

+
to_hex(val: bytes) → string

Converts val to its hexadecimal representation.

+
to_hex(val: int) → string

Converts val to its hexadecimal representation.

+
to_hex(val: string) → string

Converts val to its hexadecimal representation.

+
to_ip(val: string) → bytes

Converts the character string representation of an IP to its byte string representation.

+
to_uuid(val: string) → bytes

Converts the character string representation of a UUID to its byte string representation.

+
translate(input: string, find: string, replace: string) → string

In input, replaces the first character from find with the first character in replace; repeat for each character in find.

+

For example, translate('doggie', 'dog', '123'); returns 1233ie.

+
upper(val: string) → string

Converts all characters in val to their to their upper-case equivalents.

+
+ +### System info functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
cluster_logical_timestamp() → decimal

Returns the logical time of the current transaction.

+

This function is reserved for testing purposes by CockroachDB +developers and its definition may change without prior notice.

+

Note that uses of this function disable server-side optimizations and +may increase either contention or retry errors, or both.

+
crdb_internal.check_consistency(stats_only: bool, start_key: bytes, end_key: bytes) → tuple{int AS range_id, bytes AS start_key, string AS start_key_pretty, string AS status, string AS detail}

Runs a consistency check on ranges touching the specified key range. an empty start or end key is treated as the minimum and maximum possible, respectively. stats_only should only be set to false when targeting a small number of ranges to avoid overloading the cluster. Each returned row contains the range ID, the status (a roachpb.CheckConsistencyResponse_Status), and verbose detail.

+

Example usage: +SELECT * FROM crdb_internal.check_consistency(true, ‘\x02’, ‘\x04’)

+
crdb_internal.cluster_id() → uuid

Returns the cluster ID.

+
crdb_internal.cluster_name() → string

Returns the cluster name.

+
crdb_internal.encode_key(table_id: int, index_id: int, row_tuple: anyelement) → bytes

Generate the key for a row on a particular table and index.

+
crdb_internal.force_assertion_error(msg: string) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.force_error(errorCode: string, msg: string) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.force_log_fatal(msg: string) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.force_panic(msg: string) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.force_retry(val: interval) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.get_namespace_id(parent_id: int, name: string) → int
crdb_internal.get_zone_config(namespace_id: int) → bytes
crdb_internal.is_admin() → bool

Retrieves the current user’s admin status.

+
crdb_internal.lease_holder(key: bytes) → int

This function is used to fetch the leaseholder corresponding to a request key

+
crdb_internal.locality_value(key: string) → string

Returns the value of the specified locality key.

+
crdb_internal.no_constant_folding(input: anyelement) → anyelement

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.node_executable_version() → string

Returns the version of CockroachDB this node is running.

+
crdb_internal.notice(msg: string) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.num_inverted_index_entries(val: anyelement[]) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.num_inverted_index_entries(val: jsonb) → int

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.pretty_key(raw_key: bytes, skip_fields: int) → string

This function is used only by CockroachDB’s developers for testing purposes.

+
crdb_internal.range_stats(key: bytes) → jsonb

This function is used to retrieve range statistics information as a JSON object.

+
crdb_internal.round_decimal_values(val: decimal, scale: int) → decimal

This function is used internally to round decimal values during mutations.

+
crdb_internal.round_decimal_values(val: decimal[], scale: int) → decimal[]

This function is used internally to round decimal array values during mutations.

+
crdb_internal.set_vmodule(vmodule_string: string) → int

Set the equivalent of the --vmodule flag on the gateway node processing this request; it affords control over the logging verbosity of different files. Example syntax: crdb_internal.set_vmodule('recordio=2,file=1,gfs*=3'). Reset with: crdb_internal.set_vmodule(''). Raising the verbosity can severely affect performance.

+
current_database() → string

Returns the current database.

+
current_schema() → string

Returns the current schema.

+
current_schemas(include_pg_catalog: bool) → string[]

Returns the valid schemas in the search path.

+
current_user() → string

Returns the current user. This function is provided for compatibility with PostgreSQL.

+
version() → string

Returns the node’s version of CockroachDB.

+
+ +### TIMETZ functions + + + + + + + + + + + + +
Function → ReturnsDescription
current_time() → time

Returns the current transaction’s time with no time zone.

+
current_time() → timetz

Returns the current transaction’s time with time zone.

+

This function is the preferred overload and will be evaluated by default.

+
current_time(precision: int) → time

Returns the current transaction’s time with no time zone.

+
current_time(precision: int) → timetz

Returns the current transaction’s time with time zone.

+

This function is the preferred overload and will be evaluated by default.

+
localtime() → time

Returns the current transaction’s time with no time zone.

+

This function is the preferred overload and will be evaluated by default.

+
localtime() → timetz

Returns the current transaction’s time with time zone.

+
localtime(precision: int) → time

Returns the current transaction’s time with no time zone.

+

This function is the preferred overload and will be evaluated by default.

+
localtime(precision: int) → timetz

Returns the current transaction’s time with time zone.

+
+ +### TUPLE functions + + + + + +
Function → ReturnsDescription
row_to_json(row: tuple) → jsonb

Returns the row as a JSON object.

+
+ +### Compatibility functions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
format_type(type_oid: oid, typemod: int) → string

Returns the SQL name of a data type that is identified by its type OID and possibly a type modifier. Currently, the type modifier is ignored.

+
getdatabaseencoding() → string

Returns the current encoding name used by the database.

+
has_any_column_privilege(table: string, privilege: string) → bool

Returns whether or not the current user has privileges for any column of table.

+
has_any_column_privilege(table: oid, privilege: string) → bool

Returns whether or not the current user has privileges for any column of table.

+
has_any_column_privilege(user: string, table: string, privilege: string) → bool

Returns whether or not the user has privileges for any column of table.

+
has_any_column_privilege(user: string, table: oid, privilege: string) → bool

Returns whether or not the user has privileges for any column of table.

+
has_any_column_privilege(user: oid, table: string, privilege: string) → bool

Returns whether or not the user has privileges for any column of table.

+
has_any_column_privilege(user: oid, table: oid, privilege: string) → bool

Returns whether or not the user has privileges for any column of table.

+
has_column_privilege(table: string, column: int, privilege: string) → bool

Returns whether or not the current user has privileges for column.

+
has_column_privilege(table: string, column: string, privilege: string) → bool

Returns whether or not the current user has privileges for column.

+
has_column_privilege(table: oid, column: int, privilege: string) → bool

Returns whether or not the current user has privileges for column.

+
has_column_privilege(table: oid, column: string, privilege: string) → bool

Returns whether or not the current user has privileges for column.

+
has_column_privilege(user: string, table: string, column: int, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: string, table: string, column: string, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: string, table: oid, column: int, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: string, table: oid, column: string, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: oid, table: string, column: int, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: oid, table: string, column: string, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: oid, table: oid, column: int, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_column_privilege(user: oid, table: oid, column: string, privilege: string) → bool

Returns whether or not the user has privileges for column.

+
has_database_privilege(database: string, privilege: string) → bool

Returns whether or not the current user has privileges for database.

+
has_database_privilege(database: oid, privilege: string) → bool

Returns whether or not the current user has privileges for database.

+
has_database_privilege(user: string, database: string, privilege: string) → bool

Returns whether or not the user has privileges for database.

+
has_database_privilege(user: string, database: oid, privilege: string) → bool

Returns whether or not the user has privileges for database.

+
has_database_privilege(user: oid, database: string, privilege: string) → bool

Returns whether or not the user has privileges for database.

+
has_database_privilege(user: oid, database: oid, privilege: string) → bool

Returns whether or not the user has privileges for database.

+
has_foreign_data_wrapper_privilege(fdw: string, privilege: string) → bool

Returns whether or not the current user has privileges for foreign-data wrapper.

+
has_foreign_data_wrapper_privilege(fdw: oid, privilege: string) → bool

Returns whether or not the current user has privileges for foreign-data wrapper.

+
has_foreign_data_wrapper_privilege(user: string, fdw: string, privilege: string) → bool

Returns whether or not the user has privileges for foreign-data wrapper.

+
has_foreign_data_wrapper_privilege(user: string, fdw: oid, privilege: string) → bool

Returns whether or not the user has privileges for foreign-data wrapper.

+
has_foreign_data_wrapper_privilege(user: oid, fdw: string, privilege: string) → bool

Returns whether or not the user has privileges for foreign-data wrapper.

+
has_foreign_data_wrapper_privilege(user: oid, fdw: oid, privilege: string) → bool

Returns whether or not the user has privileges for foreign-data wrapper.

+
has_function_privilege(function: string, privilege: string) → bool

Returns whether or not the current user has privileges for function.

+
has_function_privilege(function: oid, privilege: string) → bool

Returns whether or not the current user has privileges for function.

+
has_function_privilege(user: string, function: string, privilege: string) → bool

Returns whether or not the user has privileges for function.

+
has_function_privilege(user: string, function: oid, privilege: string) → bool

Returns whether or not the user has privileges for function.

+
has_function_privilege(user: oid, function: string, privilege: string) → bool

Returns whether or not the user has privileges for function.

+
has_function_privilege(user: oid, function: oid, privilege: string) → bool

Returns whether or not the user has privileges for function.

+
has_language_privilege(language: string, privilege: string) → bool

Returns whether or not the current user has privileges for language.

+
has_language_privilege(language: oid, privilege: string) → bool

Returns whether or not the current user has privileges for language.

+
has_language_privilege(user: string, language: string, privilege: string) → bool

Returns whether or not the user has privileges for language.

+
has_language_privilege(user: string, language: oid, privilege: string) → bool

Returns whether or not the user has privileges for language.

+
has_language_privilege(user: oid, language: string, privilege: string) → bool

Returns whether or not the user has privileges for language.

+
has_language_privilege(user: oid, language: oid, privilege: string) → bool

Returns whether or not the user has privileges for language.

+
has_schema_privilege(schema: string, privilege: string) → bool

Returns whether or not the current user has privileges for schema.

+
has_schema_privilege(schema: oid, privilege: string) → bool

Returns whether or not the current user has privileges for schema.

+
has_schema_privilege(user: string, schema: string, privilege: string) → bool

Returns whether or not the user has privileges for schema.

+
has_schema_privilege(user: string, schema: oid, privilege: string) → bool

Returns whether or not the user has privileges for schema.

+
has_schema_privilege(user: oid, schema: string, privilege: string) → bool

Returns whether or not the user has privileges for schema.

+
has_schema_privilege(user: oid, schema: oid, privilege: string) → bool

Returns whether or not the user has privileges for schema.

+
has_sequence_privilege(sequence: string, privilege: string) → bool

Returns whether or not the current user has privileges for sequence.

+
has_sequence_privilege(sequence: oid, privilege: string) → bool

Returns whether or not the current user has privileges for sequence.

+
has_sequence_privilege(user: string, sequence: string, privilege: string) → bool

Returns whether or not the user has privileges for sequence.

+
has_sequence_privilege(user: string, sequence: oid, privilege: string) → bool

Returns whether or not the user has privileges for sequence.

+
has_sequence_privilege(user: oid, sequence: string, privilege: string) → bool

Returns whether or not the user has privileges for sequence.

+
has_sequence_privilege(user: oid, sequence: oid, privilege: string) → bool

Returns whether or not the user has privileges for sequence.

+
has_server_privilege(server: string, privilege: string) → bool

Returns whether or not the current user has privileges for foreign server.

+
has_server_privilege(server: oid, privilege: string) → bool

Returns whether or not the current user has privileges for foreign server.

+
has_server_privilege(user: string, server: string, privilege: string) → bool

Returns whether or not the user has privileges for foreign server.

+
has_server_privilege(user: string, server: oid, privilege: string) → bool

Returns whether or not the user has privileges for foreign server.

+
has_server_privilege(user: oid, server: string, privilege: string) → bool

Returns whether or not the user has privileges for foreign server.

+
has_server_privilege(user: oid, server: oid, privilege: string) → bool

Returns whether or not the user has privileges for foreign server.

+
has_table_privilege(table: string, privilege: string) → bool

Returns whether or not the current user has privileges for table.

+
has_table_privilege(table: oid, privilege: string) → bool

Returns whether or not the current user has privileges for table.

+
has_table_privilege(user: string, table: string, privilege: string) → bool

Returns whether or not the user has privileges for table.

+
has_table_privilege(user: string, table: oid, privilege: string) → bool

Returns whether or not the user has privileges for table.

+
has_table_privilege(user: oid, table: string, privilege: string) → bool

Returns whether or not the user has privileges for table.

+
has_table_privilege(user: oid, table: oid, privilege: string) → bool

Returns whether or not the user has privileges for table.

+
has_tablespace_privilege(tablespace: string, privilege: string) → bool

Returns whether or not the current user has privileges for tablespace.

+
has_tablespace_privilege(tablespace: oid, privilege: string) → bool

Returns whether or not the current user has privileges for tablespace.

+
has_tablespace_privilege(user: string, tablespace: string, privilege: string) → bool

Returns whether or not the user has privileges for tablespace.

+
has_tablespace_privilege(user: string, tablespace: oid, privilege: string) → bool

Returns whether or not the user has privileges for tablespace.

+
has_tablespace_privilege(user: oid, tablespace: string, privilege: string) → bool

Returns whether or not the user has privileges for tablespace.

+
has_tablespace_privilege(user: oid, tablespace: oid, privilege: string) → bool

Returns whether or not the user has privileges for tablespace.

+
has_type_privilege(type: string, privilege: string) → bool

Returns whether or not the current user has privileges for type.

+
has_type_privilege(type: oid, privilege: string) → bool

Returns whether or not the current user has privileges for type.

+
has_type_privilege(user: string, type: string, privilege: string) → bool

Returns whether or not the user has privileges for type.

+
has_type_privilege(user: string, type: oid, privilege: string) → bool

Returns whether or not the user has privileges for type.

+
has_type_privilege(user: oid, type: string, privilege: string) → bool

Returns whether or not the user has privileges for type.

+
has_type_privilege(user: oid, type: oid, privilege: string) → bool

Returns whether or not the user has privileges for type.

+
oid(int: int) → oid

Converts an integer to an OID.

+
pg_sleep(seconds: float) → bool

pg_sleep makes the current session’s process sleep until seconds seconds have elapsed. seconds is a value of type double precision, so fractional-second delays can be specified.

+
+ diff --git a/_includes/v20.2/sql/import-default-value.md b/_includes/v20.2/sql/import-default-value.md new file mode 100644 index 00000000000..4a88ba003fb --- /dev/null +++ b/_includes/v20.2/sql/import-default-value.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +Column values cannot be generated by [`DEFAULT`](default-value.html) when importing; an import must include a value for every column specified in the `IMPORT` statement. To use `DEFAULT` values, your file must contain values for the column upon import, or you can [add the column](add-column.html) or [alter the column](alter-column.html#set-or-change-a-default-value) after the table has been imported. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/import-into-default-value.md b/_includes/v20.2/sql/import-into-default-value.md new file mode 100644 index 00000000000..8c23d6e3de4 --- /dev/null +++ b/_includes/v20.2/sql/import-into-default-value.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +Column values cannot be generated by [`DEFAULT`](default-value.html) when importing; an import must include a value for every column specified in the `IMPORT INTO` statement. To use `DEFAULT` values, your file must contain values for the column upon import, or you can [add the column](add-column.html) or [alter the column](alter-column.html#set-or-change-a-default-value) after the table has been imported. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/movr-start-nodes.md b/_includes/v20.2/sql/movr-start-nodes.md new file mode 100644 index 00000000000..18f763bb09b --- /dev/null +++ b/_includes/v20.2/sql/movr-start-nodes.md @@ -0,0 +1,6 @@ +Run [`cockroach demo`](cockroach-demo.html) with the [`--nodes`](cockroach-demo.html#flags) and [`--demo-locality`](cockroach-demo.html#flags) flags This command opens an interactive SQL shell to a temporary, multi-node in-memory cluster with the `movr` database preloaded and set as the [current database](sql-name-resolution.html#current-database). + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach demo --nodes=3 --demo-locality=region=us-east1:region=us-central1:region=us-west1 + ~~~ diff --git a/_includes/v20.2/sql/movr-start.md b/_includes/v20.2/sql/movr-start.md new file mode 100644 index 00000000000..7f8bca7d2c0 --- /dev/null +++ b/_includes/v20.2/sql/movr-start.md @@ -0,0 +1,54 @@ +- Run [`cockroach demo`](cockroach-demo.html) to start a temporary, in-memory cluster with the `movr` dataset preloaded: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach demo + ~~~ + +- Load the `movr` dataset into a persistent local cluster and open an interactive SQL shell: + 1. Start a [secure](secure-a-cluster.html) or [insecure](start-a-local-cluster.html) local cluster. + 1. Use [`cockroach workload`](cockroach-workload.html) to load the `movr` dataset: + +
+ + +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init movr 'postgresql://root@localhost:26257?sslcert=certs%2Fclient.root.crt&sslkey=certs%2Fclient.root.key&sslmode=verify-full&sslrootcert=certs%2Fca.crt' + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init movr 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ +
+ 1. Use [`cockroach sql`](cockroach-sql.html) to open an interactive SQL shell and set `movr` as the [current database](sql-name-resolution.html#current-database): + +
+ {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs --host=localhost:26257 + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > USE movr; + ~~~ +
+ +
+ {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost:26257 + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > USE movr; + ~~~ +
diff --git a/_includes/v20.2/sql/movr-statements-geo-partitioned-replicas.md b/_includes/v20.2/sql/movr-statements-geo-partitioned-replicas.md new file mode 100644 index 00000000000..70b2f544161 --- /dev/null +++ b/_includes/v20.2/sql/movr-statements-geo-partitioned-replicas.md @@ -0,0 +1,10 @@ +### Setup + +The following examples use MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +To follow along, run [`cockroach demo`](cockroach-demo.html) with the [`--geo-partitioned-replicas` flag](cockroach-demo.html#run-cockroach-demo-with-geo-partitioned-replicas). This command opens an interactive SQL shell to a temporary, 9-node in-memory cluster with the [Geo-Partitioned Replicas Topology](topology-geo-partitioned-replicas.html) applied to the `movr` database. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo --geo-partitioned-replicas +~~~ diff --git a/_includes/v20.2/sql/movr-statements-nodes.md b/_includes/v20.2/sql/movr-statements-nodes.md new file mode 100644 index 00000000000..1c8f2b4daf8 --- /dev/null +++ b/_includes/v20.2/sql/movr-statements-nodes.md @@ -0,0 +1,10 @@ +### Setup + +The following examples use MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +To follow along, run [`cockroach demo`](cockroach-demo.html) with the [`--nodes`](cockroach-demo.html#flags) and [`--demo-locality`](cockroach-demo.html#flags) flags. This command opens an interactive SQL shell to a temporary, multi-node in-memory cluster with the `movr` database preloaded and set as the [current database](sql-name-resolution.html#current-database). + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo --nodes=3 --demo-locality=region=us-east1:region=us-central1:region=us-west1 +~~~ diff --git a/_includes/v20.2/sql/movr-statements-partitioning.md b/_includes/v20.2/sql/movr-statements-partitioning.md new file mode 100644 index 00000000000..ec9d333de2c --- /dev/null +++ b/_includes/v20.2/sql/movr-statements-partitioning.md @@ -0,0 +1,10 @@ +The following examples use MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +To follow along with the examples below, open a new terminal and run [`cockroach demo`](cockroach-demo.html) with the [`--nodes`](cockroach-demo.html#flags) and [`--demo-locality`](cockroach-demo.html#flags) flags. This command opens an interactive SQL shell to a temporary, multi-node in-memory cluster with the `movr` database preloaded and set as the [current database](sql-name-resolution.html#current-database). + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo \ +--nodes=9 \ +--demo-locality=region=us-east1:region=us-east1:region=us-east1:region=us-central1:region=us-central1:region=us-central1:region=us-west1:region=us-west1:region=us-west1 +~~~ diff --git a/_includes/v20.2/sql/movr-statements.md b/_includes/v20.2/sql/movr-statements.md new file mode 100644 index 00000000000..656f92346dc --- /dev/null +++ b/_includes/v20.2/sql/movr-statements.md @@ -0,0 +1,10 @@ +### Setup + +The following examples use MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +To follow along, run [`cockroach demo`](cockroach-demo.html) to start a temporary, in-memory cluster with the `movr` dataset preloaded: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo +~~~ diff --git a/_includes/v20.2/sql/operators.md b/_includes/v20.2/sql/operators.md new file mode 100644 index 00000000000..fa38394675f --- /dev/null +++ b/_includes/v20.2/sql/operators.md @@ -0,0 +1,503 @@ + + + + + +
#Return
int # intint
varbit # varbitvarbit
+ + + + +
#>Return
jsonb #> string[]jsonb
+ + + + +
#>>Return
jsonb #>> string[]string
+ + + + + + + + +
%Return
decimal % decimaldecimal
decimal % intdecimal
float % floatfloat
int % decimaldecimal
int % intint
+ + + + + + +
&Return
inet & inetinet
int & intint
varbit & varbitvarbit
+ + + + + +
&&Return
anyelement && anyelementbool
inet && inetbool
+ + + + + + + + + + + + + + +
*Return
decimal * decimaldecimal
decimal * intdecimal
decimal * intervalinterval
float * floatfloat
float * intervalinterval
int * decimaldecimal
int * intint
int * intervalinterval
interval * decimalinterval
interval * floatinterval
interval * intinterval
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
+Return
date + intdate
date + intervaltimestamp
date + timetimestamp
date + timetztimestamptz
decimal + decimaldecimal
decimal + intdecimal
float + floatfloat
inet + intinet
int + datedate
int + decimaldecimal
int + inetinet
int + intint
interval + datetimestamp
interval + intervalinterval
interval + timetime
interval + timestamptimestamp
interval + timestamptztimestamptz
interval + timetztimetz
time + datetimestamp
time + intervaltime
timestamp + intervaltimestamp
timestamptz + intervaltimestamptz
timetz + datetimestamptz
timetz + intervaltimetz
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-Return
-decimaldecimal
-floatfloat
-intint
-intervalinterval
date - dateint
date - intdate
date - intervaltimestamp
date - timetimestamp
decimal - decimaldecimal
decimal - intdecimal
float - floatfloat
inet - inetint
inet - intinet
int - decimaldecimal
int - intint
interval - intervalinterval
jsonb - intjsonb
jsonb - stringjsonb
jsonb - string[]jsonb
time - intervaltime
time - timeinterval
timestamp - intervaltimestamp
timestamp - timestampinterval
timestamp - timestamptzinterval
timestamptz - intervaltimestamptz
timestamptz - timestampinterval
timestamptz - timestamptzinterval
timetz - intervaltimetz
+ + + + + +
->Return
jsonb -> intjsonb
jsonb -> stringjsonb
+ + + + + +
->>Return
jsonb ->> intstring
jsonb ->> stringstring
+ + + + + + + + + + +
/Return
decimal / decimaldecimal
decimal / intdecimal
float / floatfloat
int / decimaldecimal
int / intdecimal
interval / floatinterval
interval / intinterval
+ + + + + + + + +
//Return
decimal // decimaldecimal
decimal // intdecimal
float // floatfloat
int // decimaldecimal
int // intint
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
<Return
bool < boolbool
bool[] < bool[]bool
bytes < bytesbool
bytes[] < bytes[]bool
collatedstring < collatedstringbool
date < datebool
date < timestampbool
date < timestamptzbool
date[] < date[]bool
decimal < decimalbool
decimal < floatbool
decimal < intbool
decimal[] < decimal[]bool
float < decimalbool
float < floatbool
float < intbool
float[] < float[]bool
inet < inetbool
inet[] < inet[]bool
int < decimalbool
int < floatbool
int < intbool
int[] < int[]bool
interval < intervalbool
interval[] < interval[]bool
jsonb < jsonbbool
oid < oidbool
string < stringbool
string[] < string[]bool
time < timebool
time < timetzbool
time[] < time[]bool
timestamp < datebool
timestamp < timestampbool
timestamp < timestamptzbool
timestamp[] < timestamp[]bool
timestamptz < datebool
timestamptz < timestampbool
timestamptz < timestamptzbool
timestamptz < timestamptzbool
timetz < timebool
timetz < timetzbool
tuple < tuplebool
uuid < uuidbool
uuid[] < uuid[]bool
varbit < varbitbool
+ + + + + + +
<<Return
inet << inetbool
int << intint
varbit << intvarbit
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
<=Return
bool <= boolbool
bool[] <= bool[]bool
bytes <= bytesbool
bytes[] <= bytes[]bool
collatedstring <= collatedstringbool
date <= datebool
date <= timestampbool
date <= timestamptzbool
date[] <= date[]bool
decimal <= decimalbool
decimal <= floatbool
decimal <= intbool
decimal[] <= decimal[]bool
float <= decimalbool
float <= floatbool
float <= intbool
float[] <= float[]bool
inet <= inetbool
inet[] <= inet[]bool
int <= decimalbool
int <= floatbool
int <= intbool
int[] <= int[]bool
interval <= intervalbool
interval[] <= interval[]bool
jsonb <= jsonbbool
oid <= oidbool
string <= stringbool
string[] <= string[]bool
time <= timebool
time <= timetzbool
time[] <= time[]bool
timestamp <= datebool
timestamp <= timestampbool
timestamp <= timestamptzbool
timestamp[] <= timestamp[]bool
timestamptz <= datebool
timestamptz <= timestampbool
timestamptz <= timestamptzbool
timestamptz <= timestamptzbool
timetz <= timebool
timetz <= timetzbool
tuple <= tuplebool
uuid <= uuidbool
uuid[] <= uuid[]bool
varbit <= varbitbool
+ + + + + +
<@Return
anyelement <@ anyelementbool
jsonb <@ jsonbbool
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
=Return
bool = boolbool
bool[] = bool[]bool
bytes = bytesbool
bytes[] = bytes[]bool
collatedstring = collatedstringbool
date = datebool
date = timestampbool
date = timestamptzbool
date[] = date[]bool
decimal = decimalbool
decimal = floatbool
decimal = intbool
decimal[] = decimal[]bool
float = decimalbool
float = floatbool
float = intbool
float[] = float[]bool
inet = inetbool
inet[] = inet[]bool
int = decimalbool
int = floatbool
int = intbool
int[] = int[]bool
interval = intervalbool
interval[] = interval[]bool
jsonb = jsonbbool
oid = oidbool
string = stringbool
string[] = string[]bool
time = timebool
time = timetzbool
time[] = time[]bool
timestamp = datebool
timestamp = timestampbool
timestamp = timestamptzbool
timestamp[] = timestamp[]bool
timestamptz = datebool
timestamptz = timestampbool
timestamptz = timestamptzbool
timestamptz = timestamptzbool
timetz = timebool
timetz = timetzbool
tuple = tuplebool
uuid = uuidbool
uuid[] = uuid[]bool
varbit = varbitbool
+ + + + + + +
>>Return
inet >> inetbool
int >> intint
varbit >> intvarbit
+ + + + +
?Return
jsonb ? stringbool
+ + + + +
?&Return
jsonb ?& string[]bool
+ + + + +
?|Return
jsonb ?| string[]bool
+ + + + + +
@>Return
anyelement @> anyelementbool
jsonb @> jsonbbool
+ + + + +
ILIKEReturn
string ILIKE stringbool
+ + + + + + + + + + + + + + + + + + + + + + +
INReturn
bool IN tuplebool
bytes IN tuplebool
collatedstring IN tuplebool
date IN tuplebool
decimal IN tuplebool
float IN tuplebool
inet IN tuplebool
int IN tuplebool
interval IN tuplebool
jsonb IN tuplebool
oid IN tuplebool
string IN tuplebool
time IN tuplebool
timestamp IN tuplebool
timestamptz IN tuplebool
timetz IN tuplebool
tuple IN tuplebool
uuid IN tuplebool
varbit IN tuplebool
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IS NOT DISTINCT FROMReturn
bool IS NOT DISTINCT FROM boolbool
bool[] IS NOT DISTINCT FROM bool[]bool
bytes IS NOT DISTINCT FROM bytesbool
bytes[] IS NOT DISTINCT FROM bytes[]bool
collatedstring IS NOT DISTINCT FROM collatedstringbool
date IS NOT DISTINCT FROM datebool
date IS NOT DISTINCT FROM timestampbool
date IS NOT DISTINCT FROM timestamptzbool
date[] IS NOT DISTINCT FROM date[]bool
decimal IS NOT DISTINCT FROM decimalbool
decimal IS NOT DISTINCT FROM floatbool
decimal IS NOT DISTINCT FROM intbool
decimal[] IS NOT DISTINCT FROM decimal[]bool
float IS NOT DISTINCT FROM decimalbool
float IS NOT DISTINCT FROM floatbool
float IS NOT DISTINCT FROM intbool
float[] IS NOT DISTINCT FROM float[]bool
inet IS NOT DISTINCT FROM inetbool
inet[] IS NOT DISTINCT FROM inet[]bool
int IS NOT DISTINCT FROM decimalbool
int IS NOT DISTINCT FROM floatbool
int IS NOT DISTINCT FROM intbool
int[] IS NOT DISTINCT FROM int[]bool
interval IS NOT DISTINCT FROM intervalbool
interval[] IS NOT DISTINCT FROM interval[]bool
jsonb IS NOT DISTINCT FROM jsonbbool
oid IS NOT DISTINCT FROM oidbool
string IS NOT DISTINCT FROM stringbool
string[] IS NOT DISTINCT FROM string[]bool
time IS NOT DISTINCT FROM timebool
time IS NOT DISTINCT FROM timetzbool
time[] IS NOT DISTINCT FROM time[]bool
timestamp IS NOT DISTINCT FROM datebool
timestamp IS NOT DISTINCT FROM timestampbool
timestamp IS NOT DISTINCT FROM timestamptzbool
timestamp[] IS NOT DISTINCT FROM timestamp[]bool
timestamptz IS NOT DISTINCT FROM datebool
timestamptz IS NOT DISTINCT FROM timestampbool
timestamptz IS NOT DISTINCT FROM timestamptzbool
timestamptz IS NOT DISTINCT FROM timestamptzbool
timetz IS NOT DISTINCT FROM timebool
timetz IS NOT DISTINCT FROM timetzbool
tuple IS NOT DISTINCT FROM tuplebool
unknown IS NOT DISTINCT FROM unknownbool
uuid IS NOT DISTINCT FROM uuidbool
uuid[] IS NOT DISTINCT FROM uuid[]bool
varbit IS NOT DISTINCT FROM varbitbool
+ + + + +
LIKEReturn
string LIKE stringbool
+ + + + +
SIMILAR TOReturn
string SIMILAR TO stringbool
+ + + + + + + + +
^Return
decimal ^ decimaldecimal
decimal ^ intdecimal
float ^ floatfloat
int ^ decimaldecimal
int ^ intint
+ + + + + + +
|Return
inet | inetinet
int | intint
varbit | varbitvarbit
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
||Return
bool || bool[]bool[]
bool[] || boolbool[]
bool[] || bool[]bool[]
bytes || bytesbytes
bytes || bytes[]bytes[]
bytes[] || bytesbytes[]
bytes[] || bytes[]bytes[]
date || date[]date[]
date[] || datedate[]
date[] || date[]date[]
decimal || decimal[]decimal[]
decimal[] || decimaldecimal[]
decimal[] || decimal[]decimal[]
float || float[]float[]
float[] || floatfloat[]
float[] || float[]float[]
inet || inet[]inet[]
inet[] || inetinet[]
inet[] || inet[]inet[]
int || int[]int[]
int[] || intint[]
int[] || int[]int[]
interval || interval[]interval[]
interval[] || intervalinterval[]
interval[] || interval[]interval[]
jsonb || jsonbjsonb
oid || oidoid
string || stringstring
string || string[]string[]
string[] || stringstring[]
string[] || string[]string[]
time || time[]time[]
time[] || timetime[]
time[] || time[]time[]
timestamp || timestamp[]timestamp[]
timestamp[] || timestamptimestamp[]
timestamp[] || timestamp[]timestamp[]
timestamptz || timestamptztimestamptz
timestamptz || timestamptztimestamptz
timestamptz || timestamptztimestamptz
timetz || timetztimetz
uuid || uuid[]uuid[]
uuid[] || uuiduuid[]
uuid[] || uuid[]uuid[]
varbit || varbitvarbit
+ + + + + + + +
~Return
~inetinet
~intint
~varbitvarbit
string ~ stringbool
+ + + + +
~*Return
string ~* stringbool
diff --git a/_includes/v20.2/sql/partitioning-enterprise.md b/_includes/v20.2/sql/partitioning-enterprise.md new file mode 100644 index 00000000000..e9a6f8ca1ee --- /dev/null +++ b/_includes/v20.2/sql/partitioning-enterprise.md @@ -0,0 +1 @@ +Partitioning is an enterprise-only feature. Each instance of `cockroach demo` loads a temporary enterprise license that expires after an hour. If you want to use enterprise-only features outside of a `cockroach demo` instance, [request a trial enterprise license](https://www.cockroachlabs.com/get-cockroachdb/), and then [set the license key](enterprise-licensing.html#set-a-license). diff --git a/_includes/v20.2/sql/physical-plan-url.md b/_includes/v20.2/sql/physical-plan-url.md new file mode 100644 index 00000000000..59e89af0683 --- /dev/null +++ b/_includes/v20.2/sql/physical-plan-url.md @@ -0,0 +1 @@ +The generated physical query plan is encoded into a byte string after the [fragment identifier (`#`)](https://en.wikipedia.org/wiki/Fragment_identifier) in the generated URL. The fragment is not sent to the web server; instead, the browser waits for the web server to return a `decode.html` resource, and then JavaScript on the web page decodes the fragment into a physical query plan diagram. The query plan is, therefore, not logged by a server external to the CockroachDB cluster and not exposed to the public internet. diff --git a/_includes/v20.2/sql/querying-partitions.md b/_includes/v20.2/sql/querying-partitions.md new file mode 100644 index 00000000000..b53b3a6f815 --- /dev/null +++ b/_includes/v20.2/sql/querying-partitions.md @@ -0,0 +1,163 @@ +## Querying partitions + +Similar to [indexes](indexes.html), partitions can improve query performance by limiting the numbers of rows that a query must scan. In the case of [geo-partitioned data](topology-geo-partitioned-replicas.html), partitioning can limit a query scan to data in a specific region. + +### Filtering on an indexed column + +If you filter the query of a partitioned table on a [column in the index directly following the partition prefix](indexes.html#indexing-columns), the [cost-based optimizer](cost-based-optimizer.html) creates a query plan that scans each partition in parallel, rather than performing a costly sequential scan of the entire table. + +For example, suppose that the tables in the [`movr`](movr.html) database are geo-partitioned by region, and you want to query the `users` table for information about a specific user. + +Here is the `CREATE TABLE` statement for the `users` table: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement ++------------+-------------------------------------------------------------------------------------+ + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) PARTITION BY LIST (city) ( + | PARTITION us_west VALUES IN (('seattle'), ('san francisco'), ('los angeles')), + | PARTITION us_east VALUES IN (('new york'), ('boston'), ('washington dc')), + | PARTITION europe_west VALUES IN (('amsterdam'), ('paris'), ('rome')) + | ); + | ALTER PARTITION europe_west OF INDEX movr.public.users@primary CONFIGURE ZONE USING + | constraints = '[+region=europe-west1]'; + | ALTER PARTITION us_east OF INDEX movr.public.users@primary CONFIGURE ZONE USING + | constraints = '[+region=us-east1]'; + | ALTER PARTITION us_west OF INDEX movr.public.users@primary CONFIGURE ZONE USING + | constraints = '[+region=us-west1]' +(1 row) +~~~ + +If you know the user's id, you can filter on the `id` column: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users WHERE id='00000000-0000-4000-8000-000000000000'; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+---------------+----------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 +(1 row) +~~~ + +An [`EXPLAIN`](explain.html) statement shows more detail about the cost-based optimizer's plan: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM users WHERE id='00000000-0000-4000-8000-000000000000'; +~~~ + +~~~ + tree | field | description ++------+-------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | distributed | true + | vectorized | false + scan | | + | table | users@primary + | spans | -/"amsterdam" /"amsterdam"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"amsterdam"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"amsterdam\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"boston" /"boston"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"boston"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"boston\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"los angeles" /"los angeles"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"los angeles"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"los angeles\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"new york" /"new york"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"new york"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"new york\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"paris" /"paris"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"paris"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"paris\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"rome" /"rome"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"rome"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"rome\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"san francisco" /"san francisco"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"san francisco"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"san francisco\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"seattle" /"seattle"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"seattle"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"seattle\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"washington dc" /"washington dc"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"washington dc"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"washington dc\x00"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"- + | filter | id = '00000000-0000-4000-8000-000000000000' +(6 rows) +~~~ + +Because the `id` column is in the primary index, directly after the partition prefix (`city`), the optimal query is constrained by the partitioned values. This means the query scans each partition in parallel for the unique `id` value. + +If you know the set of all possible partitioned values, adding a check constraint to the table's create statement can also improve performance. For example: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD CONSTRAINT check_city CHECK (city IN ('amsterdam','boston','los angeles','new york','paris','rome','san francisco','seattle','washington dc')); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM users WHERE id='00000000-0000-4000-8000-000000000000'; +~~~ + +~~~ + tree | field | description ++------+-------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | distributed | false + | vectorized | false + scan | | + | table | users@primary + | spans | /"amsterdam"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"amsterdam"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"boston"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"boston"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"los angeles"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"los angeles"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"new york"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"new york"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"paris"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"paris"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"rome"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"rome"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"san francisco"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"san francisco"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"seattle"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"seattle"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# /"washington dc"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"-/"washington dc"/"\x00\x00\x00\x00\x00\x00@\x00\x80\x00\x00\x00\x00\x00\x00\x00"/# + | parallel | +(6 rows) +~~~ + + +To see the performance improvement over a query that performs a full table scan, compare these queries to a query with a filter on a column that is not in the index. + +### Filtering on a non-indexed column + +Suppose that you want to query the `users` table for information about a specific user, but you only know the user's name. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users WHERE name='Robert Murphy'; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+---------------+----------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM users WHERE name='Robert Murphy'; +~~~ + +~~~ + tree | field | description ++------+-------------+------------------------+ + | distributed | true + | vectorized | false + scan | | + | table | users@primary + | spans | ALL + | filter | name = 'Robert Murphy' +(6 rows) +~~~ + +The query returns the same result, but because `name` is not an indexed column, the query performs a full table scan that spans across all partition values. + +### Filtering on an partitioned column + +If you know which partition contains the data that you are querying, using a filter (e.g. a [`WHERE` clause](select-clause.html#filter-rows)) on the column that is used for the partition can further improve performance by limiting the scan to the specific partition(s) that contain the data that you are querying. + +Now suppose that you know the user's name and location. You can query the table with a filter on the user's name and city: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM users WHERE name='Robert Murphy' AND city='new york'; +~~~ + +~~~ + tree | field | description ++------+-------------+-----------------------------------+ + | distributed | true + | vectorized | false + scan | | + | table | users@primary + | spans | /"new york"-/"new york"/PrefixEnd + | filter | name = 'Robert Murphy' +(6 rows) +~~~ + +The table returns the same results as before, but at a much lower cost, as the query scan now spans just the `new york` partition value. diff --git a/_includes/v20.2/sql/retry-savepoints.md b/_includes/v20.2/sql/retry-savepoints.md new file mode 100644 index 00000000000..6b9e78209f0 --- /dev/null +++ b/_includes/v20.2/sql/retry-savepoints.md @@ -0,0 +1 @@ +A savepoint defined with the name `cockroach_restart` is a "retry savepoint" and is used to implement [advanced client-side transaction retries](advanced-client-side-transaction-retries.html). For more information, see [Retry savepoints](advanced-client-side-transaction-retries.html#retry-savepoints). diff --git a/_includes/v20.2/sql/savepoint-ddl-rollbacks.md b/_includes/v20.2/sql/savepoint-ddl-rollbacks.md new file mode 100644 index 00000000000..57da82ae775 --- /dev/null +++ b/_includes/v20.2/sql/savepoint-ddl-rollbacks.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_danger}} +Rollbacks to savepoints over [DDL](https://en.wikipedia.org/wiki/Data_definition_language) statements are only supported if you're rolling back to a savepoint created at the beginning of the transaction. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/savepoints-and-high-priority-transactions.md b/_includes/v20.2/sql/savepoints-and-high-priority-transactions.md new file mode 100644 index 00000000000..4b77f2dd561 --- /dev/null +++ b/_includes/v20.2/sql/savepoints-and-high-priority-transactions.md @@ -0,0 +1 @@ +[`ROLLBACK TO SAVEPOINT`](rollback-transaction.html#rollback-a-nested-transaction) (for either regular savepoints or "restart savepoints" defined with `cockroach_restart`) causes a "feature not supported" error after a DDL statement in a [`HIGH PRIORITY` transaction](transactions.html#transaction-priorities), in order to avoid a transaction deadlock. For more information, see GitHub issue [#46414](https://www.github.com/cockroachdb/cockroach/issues/46414). diff --git a/_includes/v20.2/sql/savepoints-and-row-locks.md b/_includes/v20.2/sql/savepoints-and-row-locks.md new file mode 100644 index 00000000000..735c4cebbbb --- /dev/null +++ b/_includes/v20.2/sql/savepoints-and-row-locks.md @@ -0,0 +1,12 @@ +CockroachDB supports exclusive row locks. + +- In PostgreSQL, row locks are released/cancelled upon [`ROLLBACK TO SAVEPOINT`][rts]. +- In CockroachDB, row locks are preserved upon [`ROLLBACK TO SAVEPOINT`][rts]. + +This is an architectural difference in v20.2 that may or may not be lifted in a later CockroachDB version. + +The code of client applications that rely on row locks must be reviewed and possibly modified to account for this difference. In particular, if an application is relying on [`ROLLBACK TO SAVEPOINT`][rts] to release row locks and allow a concurrent transaction touching the same rows to proceed, this behavior will not work with CockroachDB. + + + +[rts]: rollback-transaction.html diff --git a/_includes/v20.2/sql/schema-changes.md b/_includes/v20.2/sql/schema-changes.md new file mode 100644 index 00000000000..04c49c2fbd2 --- /dev/null +++ b/_includes/v20.2/sql/schema-changes.md @@ -0,0 +1 @@ +- Schema changes through [`ALTER TABLE`](alter-table.html), [`DROP DATABASE`](drop-database.html), [`DROP TABLE`](drop-table.html), and [`TRUNCATE`](truncate.html) \ No newline at end of file diff --git a/_includes/v20.2/sql/select-for-update-overview.md b/_includes/v20.2/sql/select-for-update-overview.md new file mode 100644 index 00000000000..cce7c40e621 --- /dev/null +++ b/_includes/v20.2/sql/select-for-update-overview.md @@ -0,0 +1,12 @@ + The `SELECT FOR UPDATE` statement is used to order transactions by controlling concurrent access to one or more rows of a table. + +It works by locking the rows returned by a [selection query][selection], such that other transactions trying to access those rows are forced to wait for the transaction that locked the rows to finish. These other transactions are effectively put into a queue based on when they tried to read the value of the locked rows. + +Because this queueing happens during the read operation, the thrashing that would otherwise occur if multiple concurrently executing transactions attempt to `SELECT` the same data and then `UPDATE` the results of that selection is prevented. By preventing this thrashing, CockroachDB also prevents the [transaction retries][retries] that would otherwise occur. + +As a result, using `SELECT FOR UPDATE` leads to increased throughput and decreased tail latency for contended operations. + + + +[retries]: transactions.html#transaction-retries +[selection]: selection-queries.html diff --git a/_includes/v20.2/sql/set-transaction-as-of-system-time-example.md b/_includes/v20.2/sql/set-transaction-as-of-system-time-example.md new file mode 100644 index 00000000000..f4cecd43fc7 --- /dev/null +++ b/_includes/v20.2/sql/set-transaction-as-of-system-time-example.md @@ -0,0 +1,24 @@ +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET TRANSACTION AS OF SYSTEM TIME '2019-04-09 18:02:52.0+00:00'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM products; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> COMMIT; +~~~ diff --git a/_includes/v20.2/sql/settings/settings.md b/_includes/v20.2/sql/settings/settings.md new file mode 100644 index 00000000000..a974faeb832 --- /dev/null +++ b/_includes/v20.2/sql/settings/settings.md @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SettingTypeDefaultDescription
cloudstorage.gs.default.keystringif set, JSON key to use during Google Cloud Storage operations
cloudstorage.http.custom_castringcustom root CA (appended to system's default CAs) for verifying certificates when interacting with HTTPS storage
cloudstorage.timeoutduration10m0sthe timeout for import/export storage operations
cluster.organizationstringorganization name
cluster.preserve_downgrade_optionstringdisable (automatic or manual) cluster version upgrade from the specified version until reset
diagnostics.forced_sql_stat_reset.intervalduration2h0m0sinterval after which SQL statement statistics are refreshed even if not collected (should be more than diagnostics.sql_stat_reset.interval). It has a max value of 24H.
diagnostics.reporting.enabledbooleantrueenable reporting diagnostic metrics to cockroach labs
diagnostics.reporting.intervalduration1h0m0sinterval at which diagnostics data should be reported
diagnostics.sql_stat_reset.intervalduration1h0m0sinterval controlling how often SQL statement statistics should be reset (should be less than diagnostics.forced_sql_stat_reset.interval). It has a max value of 24H.
enterprise.licensestringthe encoded cluster license
external.graphite.endpointstringif nonempty, push server metrics to the Graphite or Carbon server at the specified host:port
external.graphite.intervalduration10sthe interval at which metrics are pushed to Graphite (if enabled)
kv.allocator.load_based_lease_rebalancing.enabledbooleantrueset to enable rebalancing of range leases based on load and latency
kv.allocator.load_based_rebalancingenumerationleases and replicaswhether to rebalance based on the distribution of QPS across stores [off = 0, leases = 1, leases and replicas = 2]
kv.allocator.qps_rebalance_thresholdfloat0.25minimum fraction away from the mean a store's QPS (such as queries per second) can be before it is considered overfull or underfull
kv.allocator.range_rebalance_thresholdfloat0.05minimum fraction away from the mean a store's range count can be before it is considered overfull or underfull
kv.bulk_io_write.max_ratebyte size1.0 TiBthe rate limit (bytes/sec) to use for writes to disk on behalf of bulk io ops
kv.closed_timestamp.follower_reads_enabledbooleantrueallow (all) replicas to serve consistent historical reads based on closed timestamp information
kv.protectedts.reconciliation.intervalduration5m0sthe frequency for reconciling jobs with protected timestamp records
kv.rangefeed.enabledbooleanfalseif set, rangefeed registration is enabled
kv.replication_reports.intervalduration1m0sthe frequency for generating the replication_constraint_stats, replication_stats_report and replication_critical_localities reports (set to 0 to disable)
kv.snapshot_rebalance.max_ratebyte size8.0 MiBthe rate limit (bytes/sec) to use for rebalance and upreplication snapshots
kv.snapshot_recovery.max_ratebyte size8.0 MiBthe rate limit (bytes/sec) to use for recovery snapshots
kv.transaction.max_intents_bytesinteger262144maximum number of bytes used to track locks in transactions
kv.transaction.max_refresh_spans_bytesinteger256000maximum number of bytes used to track refresh spans in serializable transactions
server.auth_log.sql_connections.enabledbooleanfalseif set, log SQL client connect and disconnect events (note: may hinder performance on loaded nodes)
server.auth_log.sql_sessions.enabledbooleanfalseif set, log SQL session login/disconnection events (note: may hinder performance on loaded nodes)
server.clock.forward_jump_check_enabledbooleanfalseif enabled, forward clock jumps > max_offset/2 will cause a panic
server.clock.persist_upper_bound_intervalduration0sthe interval between persisting the wall time upper bound of the clock. The clock does not generate a wall time greater than the persisted timestamp and will panic if it sees a wall time greater than this value. When cockroach starts, it waits for the wall time to catch-up till this persisted timestamp. This guarantees monotonic wall time across server restarts. Not setting this or setting a value of 0 disables this feature.
server.eventlog.ttlduration2160h0m0sif nonzero, event log entries older than this duration are deleted every 10m0s. Should not be lowered below 24 hours.
server.host_based_authentication.configurationstringhost-based authentication configuration to use during connection authentication
server.rangelog.ttlduration720h0m0sif nonzero, range log entries older than this duration are deleted every 10m0s. Should not be lowered below 24 hours.
server.remote_debugging.modestringlocalset to enable remote debugging, localhost-only or disable (any, local, off)
server.shutdown.drain_waitduration0sthe amount of time a server waits in an unready state before proceeding with the rest of the shutdown process
server.shutdown.query_waitduration10sthe server will wait for at least this amount of time for active queries to finish
server.time_until_store_deadduration5m0sthe time after which if there is no new gossiped information about a store, it is considered dead
server.user_login.timeoutduration10stimeout after which client authentication times out if some system range is unavailable (0 = no timeout)
server.web_session_timeoutduration168h0m0sthe duration that a newly created web session will be valid
sql.defaults.default_int_sizeinteger8the size, in bytes, of an INT type
sql.defaults.results_buffer.sizebyte size16 KiBdefault size of the buffer that accumulates results for a statement or a batch of statements before they are sent to the client. This can be overridden on an individual connection with the 'results_buffer_size' parameter. Note that auto-retries generally only happen while no results have been delivered to the client, so reducing this size can increase the number of retriable errors a client receives. On the other hand, increasing the buffer size can increase the delay until the client receives the first result row. Updating the setting only affects new connections. Setting to 0 disables any buffering.
sql.defaults.serial_normalizationenumerationrowiddefault handling of SERIAL in table definitions [rowid = 0, virtual_sequence = 1, sql_sequence = 2]
sql.distsql.max_running_flowsinteger500maximum number of concurrent flows that can be run on a node
sql.distsql.temp_storage.joinsbooleantrueset to true to enable use of disk for distributed sql joins. Note that disabling this can have negative impact on memory usage and performance.
sql.distsql.temp_storage.sortsbooleantrueset to true to enable use of disk for distributed sql sorts. Note that disabling this can have negative impact on memory usage and performance.
sql.log.slow_query.latency_thresholdduration0swhen set to non-zero, log statements whose service latency exceeds the threshold to a secondary logger on each node
sql.metrics.statement_details.dump_to_logsbooleanfalsedump collected statement statistics to node logs when periodically cleared
sql.metrics.statement_details.enabledbooleantruecollect per-statement query statistics
sql.metrics.statement_details.plan_collection.enabledbooleantrueperiodically save a logical plan for each fingerprint
sql.metrics.statement_details.plan_collection.periodduration5m0sthe time until a new logical plan is collected
sql.metrics.statement_details.thresholdduration0sminimum execution time to cause statistics to be collected
sql.metrics.transaction_details.enabledbooleantruecollect per-application transaction statistics
sql.notices.enabledbooleantrueenable notices in the server/client protocol being sent
sql.stats.automatic_collection.enabledbooleantrueautomatic statistics collection mode
sql.stats.automatic_collection.fraction_stale_rowsfloat0.2target fraction of stale rows per table that will trigger a statistics refresh
sql.stats.automatic_collection.min_stale_rowsinteger500target minimum number of stale rows per table that will trigger a statistics refresh
sql.stats.histogram_collection.enabledbooleantruehistogram collection mode
sql.stats.post_events.enabledbooleanfalseif set, an event is logged for every CREATE STATISTICS job
sql.temp_object_cleaner.cleanup_intervalduration30m0show often to clean up orphaned temporary objects
sql.trace.log_statement_executebooleanfalseset to true to enable logging of executed statements
sql.trace.session_eventlog.enabledbooleanfalseset to true to enable session tracing. Note that enabling this may have a non-trivial negative performance impact.
sql.trace.txn.enable_thresholdduration0sduration beyond which all transactions are traced (set to 0 to disable)
timeseries.storage.enabledbooleantrueif set, periodic timeseries data is stored within the cluster; disabling is not recommended unless you are storing the data elsewhere
timeseries.storage.resolution_10s.ttlduration240h0m0sthe maximum age of time series data stored at the 10 second resolution. Data older than this is subject to rollup and deletion.
timeseries.storage.resolution_30m.ttlduration2160h0m0sthe maximum age of time series data stored at the 30 minute resolution. Data older than this is subject to deletion.
trace.debug.enablebooleanfalseif set, traces for recent requests can be seen in the /debug page
trace.lightstep.tokenstringif set, traces go to Lightstep using this token
trace.zipkin.collectorstringif set, traces go to the given Zipkin instance (example: '127.0.0.1:9411'); ignored if trace.lightstep.token is set
versioncustom validation19.2-14set the active cluster version in the format '.'
diff --git a/_includes/v20.2/sql/unsupported-postgres-features.md b/_includes/v20.2/sql/unsupported-postgres-features.md new file mode 100644 index 00000000000..d40abbe932a --- /dev/null +++ b/_includes/v20.2/sql/unsupported-postgres-features.md @@ -0,0 +1,10 @@ +- Stored procedures and functions +- Triggers +- Events +- User-defined functions +- FULLTEXT functions and indexes +- GEOSPATIAL functions and indexes +- Drop primary key +- XML Functions +- Column-level privileges +- XA syntax diff --git a/_includes/v20.2/sql/use-import-into.md b/_includes/v20.2/sql/use-import-into.md new file mode 100644 index 00000000000..1da57d9575b --- /dev/null +++ b/_includes/v20.2/sql/use-import-into.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +To import data into an existing table, use [`IMPORT INTO`](import-into.html). +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/vectorized-support.md b/_includes/v20.2/sql/vectorized-support.md new file mode 100644 index 00000000000..42b8e196f98 --- /dev/null +++ b/_includes/v20.2/sql/vectorized-support.md @@ -0,0 +1,3 @@ +{{site.data.alerts.callout_info}} +[Vectorized execution](vectorized-execution.html) is currently not supported for this data type. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/sql/window_functions.md b/_includes/v20.2/sql/window_functions.md new file mode 100644 index 00000000000..76c476aebd3 --- /dev/null +++ b/_includes/v20.2/sql/window_functions.md @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Function → ReturnsDescription
cume_dist() → float

Calculates the relative rank of the current row: (number of rows preceding or peer with current row) / (total rows).

+
dense_rank() → int

Calculates the rank of the current row without gaps; this function counts peer groups.

+
first_value(val: bool) → bool

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: bytes) → bytes

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: date) → date

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: decimal) → decimal

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: float) → float

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: inet) → inet

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: int) → int

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: interval) → interval

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: string) → string

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: time) → time

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: timestamp) → timestamp

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: timestamptz) → timestamptz

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: uuid) → uuid

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: jsonb) → jsonb

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: oid) → oid

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: timetz) → timetz

Returns val evaluated at the row that is the first row of the window frame.

+
first_value(val: varbit) → varbit

Returns val evaluated at the row that is the first row of the window frame.

+
lag(val: bool) → bool

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: bool, n: int) → bool

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: bool, n: int, default: bool) → bool

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: bytes) → bytes

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: bytes, n: int) → bytes

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: bytes, n: int, default: bytes) → bytes

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: date) → date

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: date, n: int) → date

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: date, n: int, default: date) → date

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: decimal) → decimal

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: decimal, n: int) → decimal

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: decimal, n: int, default: decimal) → decimal

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: float) → float

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: float, n: int) → float

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: float, n: int, default: float) → float

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: inet) → inet

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: inet, n: int) → inet

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: inet, n: int, default: inet) → inet

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: int) → int

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: int, n: int) → int

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: int, n: int, default: int) → int

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: interval) → interval

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: interval, n: int) → interval

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: interval, n: int, default: interval) → interval

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: string) → string

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: string, n: int) → string

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: string, n: int, default: string) → string

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: time) → time

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: time, n: int) → time

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: time, n: int, default: time) → time

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: timestamp) → timestamp

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: timestamp, n: int) → timestamp

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: timestamp, n: int, default: timestamp) → timestamp

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: timestamptz) → timestamptz

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: timestamptz, n: int) → timestamptz

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: timestamptz, n: int, default: timestamptz) → timestamptz

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: uuid) → uuid

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: uuid, n: int) → uuid

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: uuid, n: int, default: uuid) → uuid

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: jsonb) → jsonb

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: jsonb, n: int) → jsonb

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: jsonb, n: int, default: jsonb) → jsonb

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: oid) → oid

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: oid, n: int) → oid

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: oid, n: int, default: oid) → oid

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: timetz) → timetz

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: timetz, n: int) → timetz

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: timetz, n: int, default: timetz) → timetz

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lag(val: varbit) → varbit

Returns val evaluated at the previous row within current row’s partition; if there is no such row, instead returns null.

+
lag(val: varbit, n: int) → varbit

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lag(val: varbit, n: int, default: varbit) → varbit

Returns val evaluated at the row that is n rows before the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
last_value(val: bool) → bool

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: bytes) → bytes

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: date) → date

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: decimal) → decimal

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: float) → float

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: inet) → inet

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: int) → int

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: interval) → interval

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: string) → string

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: time) → time

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: timestamp) → timestamp

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: timestamptz) → timestamptz

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: uuid) → uuid

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: jsonb) → jsonb

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: oid) → oid

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: timetz) → timetz

Returns val evaluated at the row that is the last row of the window frame.

+
last_value(val: varbit) → varbit

Returns val evaluated at the row that is the last row of the window frame.

+
lead(val: bool) → bool

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: bool, n: int) → bool

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: bool, n: int, default: bool) → bool

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: bytes) → bytes

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: bytes, n: int) → bytes

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: bytes, n: int, default: bytes) → bytes

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: date) → date

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: date, n: int) → date

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: date, n: int, default: date) → date

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: decimal) → decimal

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: decimal, n: int) → decimal

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: decimal, n: int, default: decimal) → decimal

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: float) → float

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: float, n: int) → float

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: float, n: int, default: float) → float

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: inet) → inet

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: inet, n: int) → inet

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: inet, n: int, default: inet) → inet

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: int) → int

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: int, n: int) → int

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: int, n: int, default: int) → int

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: interval) → interval

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: interval, n: int) → interval

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: interval, n: int, default: interval) → interval

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: string) → string

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: string, n: int) → string

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: string, n: int, default: string) → string

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: time) → time

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: time, n: int) → time

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: time, n: int, default: time) → time

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: timestamp) → timestamp

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: timestamp, n: int) → timestamp

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: timestamp, n: int, default: timestamp) → timestamp

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: timestamptz) → timestamptz

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: timestamptz, n: int) → timestamptz

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: timestamptz, n: int, default: timestamptz) → timestamptz

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: uuid) → uuid

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: uuid, n: int) → uuid

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: uuid, n: int, default: uuid) → uuid

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: jsonb) → jsonb

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: jsonb, n: int) → jsonb

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: jsonb, n: int, default: jsonb) → jsonb

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: oid) → oid

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: oid, n: int) → oid

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: oid, n: int, default: oid) → oid

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: timetz) → timetz

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: timetz, n: int) → timetz

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: timetz, n: int, default: timetz) → timetz

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
lead(val: varbit) → varbit

Returns val evaluated at the following row within current row’s partition; if there is no such row, instead returns null.

+
lead(val: varbit, n: int) → varbit

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such row, instead returns null. n is evaluated with respect to the current row.

+
lead(val: varbit, n: int, default: varbit) → varbit

Returns val evaluated at the row that is n rows after the current row within its partition; if there is no such, row, instead returns default (which must be of the same type as val). Both n and default are evaluated with respect to the current row.

+
nth_value(val: bool, n: int) → bool

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: bytes, n: int) → bytes

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: date, n: int) → date

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: decimal, n: int) → decimal

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: float, n: int) → float

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: inet, n: int) → inet

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: int, n: int) → int

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: interval, n: int) → interval

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: string, n: int) → string

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: time, n: int) → time

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: timestamp, n: int) → timestamp

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: timestamptz, n: int) → timestamptz

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: uuid, n: int) → uuid

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: jsonb, n: int) → jsonb

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: oid, n: int) → oid

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: timetz, n: int) → timetz

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
nth_value(val: varbit, n: int) → varbit

Returns val evaluated at the row that is the nth row of the window frame (counting from 1); null if no such row.

+
ntile(n: int) → int

Calculates an integer ranging from 1 to n, dividing the partition as equally as possible.

+
percent_rank() → float

Calculates the relative rank of the current row: (rank - 1) / (total rows - 1).

+
rank() → int

Calculates the rank of the current row with gaps; same as row_number of its first peer.

+
row_number() → int

Calculates the number of the current row within its partition, counting from 1.

+
+ diff --git a/_includes/v20.2/start-in-docker/mac-linux-steps.md b/_includes/v20.2/start-in-docker/mac-linux-steps.md new file mode 100644 index 00000000000..4b7a762bf68 --- /dev/null +++ b/_includes/v20.2/start-in-docker/mac-linux-steps.md @@ -0,0 +1,239 @@ +## Step 1. Create a bridge network + +Since you'll be running multiple Docker containers on a single host, with one CockroachDB node per container, you need to create what Docker refers to as a [bridge network](https://docs.docker.com/engine/userguide/networking/#/a-bridge-network). The bridge network will enable the containers to communicate as a single cluster while keeping them isolated from external networks. + +{% include copy-clipboard.html %} +~~~ shell +$ docker network create -d bridge roachnet +~~~ + +We've used `roachnet` as the network name here and in subsequent steps, but feel free to give your network any name you like. + +## Step 2. Start the cluster + +1. Start the first node: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker run -d \ + --name=roach1 \ + --hostname=roach1 \ + --net=roachnet \ + -p 26257:26257 -p 8080:8080 \ + -v "${PWD}/cockroach-data/roach1:/cockroach/cockroach-data" \ + {{page.release_info.docker_image}}:{{page.release_info.version}} start \ + --insecure \ + --join=roach1,roach2,roach3 + ~~~ + +2. This command creates a container and starts the first CockroachDB node inside it. Take a moment to understand each part: + - `docker run`: The Docker command to start a new container. + - `-d`: This flag runs the container in the background so you can continue the next steps in the same shell. + - `--name`: The name for the container. This is optional, but a custom name makes it significantly easier to reference the container in other commands, for example, when opening a Bash session in the container or stopping the container. + - `--hostname`: The hostname for the container. You will use this to join other containers/nodes to the cluster. + - `--net`: The bridge network for the container to join. See step 1 for more details. + - `-p 26257:26257 -p 8080:8080`: These flags map the default port for inter-node and client-node communication (`26257`) and the default port for HTTP requests to the Admin UI (`8080`) from the container to the host. This enables inter-container communication and makes it possible to call up the Admin UI from a browser. + - `-v "${PWD}/cockroach-data/roach1:/cockroach/cockroach-data"`: This flag mounts a host directory as a data volume. This means that data and logs for this node will be stored in `${PWD}/cockroach-data/roach1` on the host and will persist after the container is stopped or deleted. For more details, see Docker's Bind Mounts topic. + - `{{page.release_info.docker_image}}:{{page.release_info.version}} start --insecure --join`: The CockroachDB command to [start a node](cockroach-start.html) in the container in insecure mode. The `--join` flag specifies the `hostname` of each node that will initially comprise your cluster. Otherwise, all [`cockroach start`](cockroach-start.html) defaults are accepted. Note that since each node is in a unique container, using identical default ports won’t cause conflicts. + +3. Start two more nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker run -d \ + --name=roach2 \ + --hostname=roach2 \ + --net=roachnet \ + -v "${PWD}/cockroach-data/roach2:/cockroach/cockroach-data" \ + {{page.release_info.docker_image}}:{{page.release_info.version}} start \ + --insecure \ + --join=roach1,roach2,roach3 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ docker run -d \ + --name=roach3 \ + --hostname=roach3 \ + --net=roachnet \ + -v "${PWD}/cockroach-data/roach3:/cockroach/cockroach-data" \ + {{page.release_info.docker_image}}:{{page.release_info.version}} start \ + --insecure \ + --join=roach1,roach2,roach3 + ~~~ + +4. Perform a one-time initialization of the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker exec -it roach1 ./cockroach init --insecure + ~~~ + + You'll see the following message: + + ~~~ + Cluster successfully initialized + ~~~ + + At this point, each node also prints helpful [startup details](cockroach-start.html#standard-output) to its log. For example, the following command retrieves node 1's startup details: + + {% include copy-clipboard.html %} + ~~~ shell + $ grep 'node starting' cockroach-data/roach1/logs/cockroach.log -A 11 + ~~~ + + The output will look something like this: + + ~~~ + CockroachDB node starting at {{page.release_info.start_time}} + build: CCL {{page.release_info.version}} @ {{page.release_info.build_time}} (go1.12.6) + webui: http://roach1:8080 + sql: postgresql://root@roach1:26257?sslmode=disable + client flags: /cockroach/cockroach --host=roach1:26257 --insecure + logs: /cockroach/cockroach-data/logs + temp dir: /cockroach/cockroach-data/cockroach-temp273641911 + external I/O path: /cockroach/cockroach-data/extern + store[0]: path=/cockroach/cockroach-data + status: initialized new cluster + clusterID: 1a705c26-e337-4b09-95a6-6e5a819f9eec + nodeID: 1 + ~~~ + +## Step 3. Use the built-in SQL client + +Now that your cluster is live, you can use any node as a SQL gateway. To test this out, let's use the `docker exec` command to start the [built-in SQL shell](cockroach-sql.html) in the first container. + +1. Start the SQL shell in the first container: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker exec -it roach1 ./cockroach sql --insecure + ~~~ + +2. Run some basic [CockroachDB SQL statements](learn-cockroachdb-sql.html): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE bank; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bank.accounts VALUES (1, 1000.50); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + id | balance + +----+---------+ + 1 | 1000.50 + (1 row) + ~~~ + +3. Now exit the SQL shell on node 1 and open a new shell on node 2: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ docker exec -it roach2 ./cockroach sql --insecure + ~~~ + +4. Run the same `SELECT` query as before: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + id | balance + +----+---------+ + 1 | 1000.50 + (1 row) + ~~~ + + As you can see, node 1 and node 2 behaved identically as SQL gateways. + +5. Exit the SQL shell on node 2: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +## Step 4. Run a sample workload + +CockroachDB also comes with a number of [built-in workloads](cockroach-workload.html) for simulating client traffic. Let's the workload based on CockroachDB's sample vehicle-sharing application, [MovR](movr.html). + +1. Load the initial dataset: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker exec -it roach1 ./cockroach workload init movr \ + 'postgresql://root@roach1:26257?sslmode=disable' + ~~~ + +2. Run the workload for 5 minutes: + + {% include copy-clipboard.html %} + ~~~ shell + $ docker exec -it roach1 ./cockroach workload run movr \ + --duration=5m \ + 'postgresql://root@roach1:26257?sslmode=disable' + ~~~ + +## Step 5. Access the Admin UI + +The CockroachDB [Admin UI](admin-ui-overview.html) gives you insight into the overall health of your cluster as well as the performance of the client workload. + +1. When you started the first container/node, you mapped the node's default HTTP port `8080` to port `8080` on the host, so go to http://localhost:8080. + +2. On the [**Cluster Overview**](admin-ui-cluster-overview-page.html), notice that three nodes are live, with an identical replica count on each node: + + CockroachDB Admin UI + + This demonstrates CockroachDB's [automated replication](demo-data-replication.html) of data via the Raft consensus protocol. + + {{site.data.alerts.callout_info}} + Capacity metrics can be incorrect when running multiple nodes on a single machine. For more details, see this [limitation](known-limitations.html#available-capacity-metric-in-the-admin-ui). + {{site.data.alerts.end}} + +3. Click [**Metrics**](admin-ui-overview-dashboard.html) to access a variety of time series dashboards, including graphs of SQL queries and service latency over time: + + CockroachDB Admin UI + +4. Use the [**Databases**](admin-ui-databases-page.html), [**Statements**](admin-ui-statements-page.html), and [**Jobs**](admin-ui-jobs-page.html) pages to view details about your databases and tables, to assess the performance of specific queries, and to monitor the status of long-running operations like schema changes, respectively. + +## Step 6. Stop the cluster + +Use the `docker stop` and `docker rm` commands to stop and remove the containers (and therefore the cluster): + +{% include copy-clipboard.html %} +~~~ shell +$ docker stop roach1 roach2 roach3 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ docker rm roach1 roach2 roach3 +~~~ + +If you do not plan to restart the cluster, you may want to remove the nodes' data stores: + +{% include copy-clipboard.html %} +~~~ shell +$ rm -rf cockroach-data +~~~ diff --git a/_includes/v20.2/topology-patterns/fundamentals.md b/_includes/v20.2/topology-patterns/fundamentals.md new file mode 100644 index 00000000000..a4a63ecfbef --- /dev/null +++ b/_includes/v20.2/topology-patterns/fundamentals.md @@ -0,0 +1,6 @@ +- Multi-region topology patterns are almost always table-specific. If you haven't already, [review the full range of patterns](topology-patterns.html#multi-region-patterns) to ensure you choose the right one for each of your tables. +- Review how data is replicated and distributed across a cluster, and how this affects performance. It is especially important to understand the concept of the "leaseholder". For a summary, see [Reads and Writes in CockroachDB](architecture/reads-and-writes-overview.html). For a deeper dive, see the [CockroachDB Architecture](architecture/overview.html) documentation. +- Review the concept of [locality](cockroach-start.html#locality), which makes CockroachDB aware of the location of nodes and able to intelligently place and balance data based on how you define [replication controls](configure-replication-zones.html). +- Review the recommendations and requirements in our [Production Checklist](recommended-production-settings.html). +- This topology doesn't account for hardware specifications, so be sure to follow our [hardware recommendations](recommended-production-settings.html#hardware) and perform a POC to size hardware for your use case. +- Adopt relevant [SQL Best Practices](performance-best-practices-overview.html) to ensure optimal performance. diff --git a/_includes/v20.2/topology-patterns/multi-region-cluster-setup.md b/_includes/v20.2/topology-patterns/multi-region-cluster-setup.md new file mode 100644 index 00000000000..3dde4aec70c --- /dev/null +++ b/_includes/v20.2/topology-patterns/multi-region-cluster-setup.md @@ -0,0 +1,29 @@ +Each [multi-region topology pattern](topology-patterns.html#multi-region-patterns) assumes the following setup: + +Multi-region hardware setup + +#### Hardware + +- 3 regions + +- Per region, 3+ AZs with 3+ VMs evenly distributed across them + +- Region-specific app instances and load balancers + - Each load balancer redirects to CockroachDB nodes in its region. + - When CockroachDB nodes are unavailable in a region, the load balancer redirects to nodes in other regions. + +#### Cluster + +Each node is started with the [`--locality`](cockroach-start.html#locality) flag specifying its region and AZ combination. For example, the following command starts a node in the west1 AZ of the us-west region: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--locality=region=us-west,zone=west1 \ +--certs-dir=certs \ +--advertise-addr= \ +--join=:26257,:26257,:26257 \ +--cache=.25 \ +--max-sql-memory=.25 \ +--background +~~~ diff --git a/_includes/v20.2/topology-patterns/see-also.md b/_includes/v20.2/topology-patterns/see-also.md new file mode 100644 index 00000000000..03844ca34fd --- /dev/null +++ b/_includes/v20.2/topology-patterns/see-also.md @@ -0,0 +1,11 @@ +- [Topology Patterns Overview](topology-patterns.html) + + - Single-region + - [Development](topology-development.html) + - [Basic Production](topology-basic-production.html) + + - Multi-region + - [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) + - [Geo-Partitioned Leaseholders](topology-geo-partitioned-leaseholders.html) + - [Duplicate Indexes](topology-duplicate-indexes.html) + - [Follow-the-Workload](topology-follow-the-workload.html) diff --git a/_includes/v20.2/zone-configs/constrain-leaseholders-to-specific-datacenters.md b/_includes/v20.2/zone-configs/constrain-leaseholders-to-specific-datacenters.md new file mode 100644 index 00000000000..eb281fbc1f3 --- /dev/null +++ b/_includes/v20.2/zone-configs/constrain-leaseholders-to-specific-datacenters.md @@ -0,0 +1,32 @@ +In addition to [constraining replicas to specific datacenters](configure-replication-zones.html#per-replica-constraints-to-specific-datacenters), you may also specify preferences for where the range's leaseholders should be placed. This can result in increased performance in some scenarios. + +The [`ALTER TABLE ... CONFIGURE ZONE`](configure-zone.html) statement below requires that the cluster try to place the ranges' leaseholders in zone `us-east1`; if that is not possible, it will try to place them in zone `us-west1`. + +For more information about how the `lease_preferences` field works, see its description in the [Replication zone variables](configure-replication-zones.html#replication-zone-variables) section. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users CONFIGURE ZONE USING num_replicas = 3, constraints = '{"+region=us-east1": 1, "+region=us-west1": 1}', lease_preferences = '[[+region=us-east1], [+region=us-west1]]'; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR TABLE users; +~~~ + +~~~ + target | raw_config_sql ++-------------+--------------------------------------------------------------------+ + TABLE users | ALTER TABLE users CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 100000, + | num_replicas = 3, + | constraints = '{+region=us-east1: 1, +region=us-west1: 1}', + | lease_preferences = '[[+region=us-east1], [+region=us-west1]]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-database.md b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-database.md new file mode 100644 index 00000000000..8cbf3e4e7de --- /dev/null +++ b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-database.md @@ -0,0 +1,28 @@ +To control replication for a specific database, use the `ALTER DATABASE ... CONFIGURE ZONE` statement to define the relevant values (other values will be inherited from the parent zone): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER DATABASE movr CONFIGURE ZONE USING num_replicas = 5, gc.ttlseconds = 100000; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR DATABASE movr; +~~~ + +~~~ + target | raw_config_sql ++---------------+------------------------------------------+ + DATABASE movr | ALTER DATABASE movr CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 100000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-secondary-index.md b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-secondary-index.md new file mode 100644 index 00000000000..5d43dc009f9 --- /dev/null +++ b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-secondary-index.md @@ -0,0 +1,42 @@ +{{site.data.alerts.callout_success}} +The [Cost-based Optimizer](cost-based-optimizer.html) can take advantage of replication zones for secondary indexes when optimizing queries. For more information, see [Cost-based optimizer - preferring the nearest index](cost-based-optimizer.html#preferring-the-nearest-index). +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +This is an [enterprise-only](enterprise-licensing.html) feature. +{{site.data.alerts.end}} + +The [secondary indexes](indexes.html) on a table will automatically use the replication zone for the table. However, with an enterprise license, you can add distinct replication zones for secondary indexes. + +To control replication for a specific secondary index, use the `ALTER INDEX ... CONFIGURE ZONE` statement to define the relevant values (other values will be inherited from the parent zone). + +{{site.data.alerts.callout_success}} +To get the name of a secondary index, which you need for the `CONFIGURE ZONE` statement, use the [`SHOW INDEX`](show-index.html) or [`SHOW CREATE TABLE`](show-create.html) statements. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> ALTER INDEX vehicles@vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING num_replicas = 5, gc.ttlseconds = 100000; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR INDEX vehicles@vehicles_auto_index_fk_city_ref_users; +~~~ + +~~~ + target | raw_config_sql ++------------------------------------------------------+---------------------------------------------------------------------------------+ + INDEX vehicles@vehicles_auto_index_fk_city_ref_users | ALTER INDEX vehicles@vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 100000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-system-range.md b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-system-range.md new file mode 100644 index 00000000000..6c6debf5da7 --- /dev/null +++ b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-system-range.md @@ -0,0 +1,41 @@ +In addition to the databases and tables that are visible via the SQL interface, CockroachDB stores internal data in what are called system ranges. CockroachDB comes with pre-configured replication zones for some of these ranges: + +Target Name | Description +----------|----------------------------- +`meta` | The "meta" ranges contain the authoritative information about the location of all data in the cluster.

These ranges must retain a majority of replicas for the cluster as a whole to remain available and historical queries are never run on them, so CockroachDB comes with a **pre-configured** `meta` replication zone with `num_replicas` set to 5 to make these ranges more resilient to node failure and a lower-than-default `gc.ttlseconds` to keep these ranges smaller for reliable performance.

If your cluster is running in multiple datacenters, it's a best practice to configure the meta ranges to have a copy in each datacenter. +`liveness` | The "liveness" range contains the authoritative information about which nodes are live at any given time.

These ranges must retain a majority of replicas for the cluster as a whole to remain available and historical queries are never run on them, so CockroachDB comes with a **pre-configured** `liveness` replication zone with `num_replicas` set to 5 to make these ranges more resilient to node failure and a lower-than-default `gc.ttlseconds` to keep these ranges smaller for reliable performance. +`system` | There are system ranges for a variety of other important internal data, including information needed to allocate new table IDs and track the status of a cluster's nodes.

These ranges must retain a majority of replicas for the cluster as a whole to remain available, so CockroachDB comes with a **pre-configured** `system` replication zone with `num_replicas` set to 5 to make these ranges more resilient to node failure. +`timeseries` | The "timeseries" ranges contain monitoring data about the cluster that powers the graphs in CockroachDB's Admin UI. If necessary, you can add a `timeseries` replication zone to control the replication of this data. + +{{site.data.alerts.callout_danger}} +Use caution when editing replication zones for system ranges, as they could cause some (or all) parts of your cluster to stop working. +{{site.data.alerts.end}} + +To control replication for one of the above sets of system ranges, use the [`ALTER RANGE ... CONFIGURE ZONE`](configure-zone.html) statement to define the relevant values (other values will be inherited from the parent zone): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER RANGE meta CONFIGURE ZONE USING num_replicas = 7; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR RANGE meta; +~~~ + +~~~ + target | raw_config_sql ++------------+---------------------------------------+ + RANGE meta | ALTER RANGE meta CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 3600, + | num_replicas = 7, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table-partition.md b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table-partition.md new file mode 100644 index 00000000000..83a0e12926c --- /dev/null +++ b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table-partition.md @@ -0,0 +1,63 @@ +{{site.data.alerts.callout_info}} +This is an [enterprise-only](enterprise-licensing.html) feature. +{{site.data.alerts.end}} + +Once [partitions have been defined for a table or a secondary index](partition-by.html), to control replication for a partition, use `ALTER PARTITION OF INDEX CONFIGURE ZONE`: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER PARTITION us_west OF INDEX vehicles@primary + CONFIGURE ZONE USING + num_replicas = 5, + constraints = '[+region=us-west1]'; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER PARTITION us_west OF INDEX vehicles@vehicles_auto_index_fk_city_ref_users + CONFIGURE ZONE USING + num_replicas = 5, + constraints = '[+region=us-west1]'; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +To define replication zones for identically named partitions of a table and its secondary indexes, you can use the `@*` syntax to save several steps: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER PARTITION us_west OF INDEX vehicles@* + CONFIGURE ZONE USING + num_replicas = 5, + constraints = '[+region=us-west1]'; +~~~ + +To view the zone configuration for a partition, use `SHOW ZONE CONFIGURATION FOR PARTITION OF INDEX `: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR PARTITION us_west OF INDEX vehicles@primary; +~~~ + +~~~ + target | raw_config_sql ++---------------------------------------------+------------------------------------------------------------------------+ + PARTITION us_west OF INDEX vehicles@primary | ALTER PARTITION us_west OF INDEX vehicles@primary CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[+region=us-west1]', + | lease_preferences = '[]' +(1 row) +~~~ + +{{site.data.alerts.callout_success}} +You can also use the [`SHOW CREATE TABLE`](show-create-table.html) statement or [`SHOW PARTITIONS`](show-partitions.html) statements to view details about all of the replication zones defined for the partitions of a table and its secondary indexes. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table.md b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table.md new file mode 100644 index 00000000000..a5e47c39bbb --- /dev/null +++ b/_includes/v20.2/zone-configs/create-a-replication-zone-for-a-table.md @@ -0,0 +1,28 @@ +To control replication for a specific table, use the `ALTER TABLE ... CONFIGURE ZONE` statement to define the relevant values (other values will be inherited from the parent zone): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users CONFIGURE ZONE USING num_replicas = 5, gc.ttlseconds = 100000; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR TABLE users; +~~~ + +~~~ + target | raw_config_sql ++-------------+----------------------------------------+ + TABLE users | ALTER TABLE users CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 100000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/edit-the-default-replication-zone.md b/_includes/v20.2/zone-configs/edit-the-default-replication-zone.md new file mode 100644 index 00000000000..a0bbbf435dd --- /dev/null +++ b/_includes/v20.2/zone-configs/edit-the-default-replication-zone.md @@ -0,0 +1,28 @@ +To edit the default replication zone, use the `ALTER RANGE ... CONFIGURE ZONE` statement to define the values you want to change (other values will remain the same): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5, gc.ttlseconds = 100000; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR RANGE default; +~~~ + +~~~ + target | raw_config_sql ++---------------+------------------------------------------+ + RANGE default | ALTER RANGE default CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 100000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/_includes/v20.2/zone-configs/remove-a-replication-zone.md b/_includes/v20.2/zone-configs/remove-a-replication-zone.md new file mode 100644 index 00000000000..b379652c8c8 --- /dev/null +++ b/_includes/v20.2/zone-configs/remove-a-replication-zone.md @@ -0,0 +1,8 @@ +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t CONFIGURE ZONE DISCARD; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ diff --git a/_includes/v20.2/zone-configs/reset-a-replication-zone.md b/_includes/v20.2/zone-configs/reset-a-replication-zone.md new file mode 100644 index 00000000000..60474c84a5d --- /dev/null +++ b/_includes/v20.2/zone-configs/reset-a-replication-zone.md @@ -0,0 +1,8 @@ +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t CONFIGURE ZONE USING DEFAULT; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ diff --git a/_includes/v20.2/zone-configs/variables.md b/_includes/v20.2/zone-configs/variables.md new file mode 100644 index 00000000000..c742ad21d2d --- /dev/null +++ b/_includes/v20.2/zone-configs/variables.md @@ -0,0 +1,14 @@ +Variable | Description +------|------------ +`range_min_bytes` | The minimum size, in bytes, for a range of data in the zone. When a range is less than this size, CockroachDB will merge it with an adjacent range.

**Default:** `16777216` (16MiB) +`range_max_bytes` | The maximum size, in bytes, for a range of data in the zone. When a range reaches this size, CockroachDB will spit it into two ranges.

**Default:** `67108864` (512 MiB) +`gc.ttlseconds` | The number of seconds overwritten values will be retained before garbage collection. Smaller values can save disk space if values are frequently overwritten; larger values increase the range allowed for `AS OF SYSTEM TIME` queries, also know as [Time Travel Queries](select-clause.html#select-historical-data-time-travel).

It is not recommended to set this below `600` (10 minutes); doing so will cause problems for long-running queries. Also, since all versions of a row are stored in a single range that never splits, it is not recommended to set this so high that all the changes to a row in that time period could add up to more than 512 MiB; such oversized ranges could contribute to the server running out of memory or other problems.

**Default:** `90000` (25 hours) +`num_replicas` | The number of replicas in the zone.

**Default:** `3`

For the `system` database and `.meta`, `.liveness`, and `.system` ranges, the default value is `5`. +`constraints` | An array of required (`+`) and/or prohibited (`-`) constraints influencing the location of replicas. See [Types of Constraints](configure-replication-zones.html#types-of-constraints) and [Scope of Constraints](configure-replication-zones.html#scope-of-constraints) for more details.

To prevent hard-to-detect typos, constraints placed on [store attributes and node localities](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes) must match the values passed to at least one node in the cluster. If not, an error is signalled.

**Default:** No constraints, with CockroachDB locating each replica on a unique node and attempting to spread replicas evenly across localities. +`lease_preferences` | An ordered list of required and/or prohibited constraints influencing the location of [leaseholders](architecture/overview.html#glossary). Whether each constraint is required or prohibited is expressed with a leading `+` or `-`, respectively. Note that lease preference constraints do not have to be shared with the `constraints` field. For example, it's valid for your configuration to define a `lease_preferences` field that does not reference any values from the `constraints` field. It's also valid to define a `lease_preferences` field with no `constraints` field at all.

If the first preference cannot be satisfied, CockroachDB will attempt to satisfy the second preference, and so on. If none of the preferences can be met, the lease will be placed using the default lease placement algorithm, which is to base lease placement decisions on how many leases each node already has, trying to make all the nodes have around the same amount.

Each value in the list can include multiple constraints. For example, the list `[[+zone=us-east-1b, +ssd], [+zone=us-east-1a], [+zone=us-east-1c, +ssd]]` means "prefer nodes with an SSD in `us-east-1b`, then any nodes in `us-east-1a`, then nodes in `us-east-1c` with an SSD."

For a usage example, see [Constrain leaseholders to specific datacenters](configure-replication-zones.html#constrain-leaseholders-to-specific-datacenters).

**Default**: No lease location preferences are applied if this field is not specified. + +{{site.data.alerts.callout_info}} +If a value is not set, new zone configurations will inherit their values from their parent zone (e.g., a partition zone inherits from the table zone), which is not necessarily `default`. + +If a variable is set to `COPY FROM PARENT` (e.g., `range_max_bytes = COPY FROM PARENT`), the variable will copy its value from its parent [replication zone](configure-replication-zones.html). The `COPY FROM PARENT` value is a convenient shortcut to use so you do not have to look up the parent's current value. For example, the `range_max_bytes` and `range_min_bytes` variables must be set together, so when editing one value, you can use `COPY FROM PARENT` for the other. Note that if the variable in the parent replication zone is changed after the child replication zone is copied, the change will not be reflected in the child zone. +{{site.data.alerts.end}} diff --git a/_includes/v20.2/zone-configs/view-all-replication-zones.md b/_includes/v20.2/zone-configs/view-all-replication-zones.md new file mode 100644 index 00000000000..58cce46cd2d --- /dev/null +++ b/_includes/v20.2/zone-configs/view-all-replication-zones.md @@ -0,0 +1,60 @@ +{% include copy-clipboard.html %} +~~~ sql +> SHOW ALL ZONE CONFIGURATIONS; +~~~ + +~~~ + target | raw_config_sql ++--------------------------------------------------+-----------------------------------------------------------------------------+ + RANGE default | ALTER RANGE default CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 3, + | constraints = '[]', + | lease_preferences = '[]' + DATABASE system | ALTER DATABASE system CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + TABLE system.public.jobs | ALTER TABLE system.public.jobs CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 600, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + RANGE meta | ALTER RANGE meta CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 3600, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + RANGE system | ALTER RANGE system CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + RANGE liveness | ALTER RANGE liveness CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 600, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + TABLE system.public.replication_constraint_stats | ALTER TABLE system.public.replication_constraint_stats CONFIGURE ZONE USING + | gc.ttlseconds = 600, + | constraints = '[]', + | lease_preferences = '[]' + TABLE system.public.replication_stats | ALTER TABLE system.public.replication_stats CONFIGURE ZONE USING + | gc.ttlseconds = 600, + | constraints = '[]', + | lease_preferences = '[]' +... +~~~ diff --git a/_includes/v20.2/zone-configs/view-the-default-replication-zone.md b/_includes/v20.2/zone-configs/view-the-default-replication-zone.md new file mode 100644 index 00000000000..50401b008ab --- /dev/null +++ b/_includes/v20.2/zone-configs/view-the-default-replication-zone.md @@ -0,0 +1,17 @@ +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATION FOR RANGE default; +~~~ + +~~~ + target | raw_config_sql ++---------------+------------------------------------------+ + RANGE default | ALTER RANGE default CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 3, + | constraints = '[]', + | lease_preferences = '[]' +(1 row) +~~~ diff --git a/images/v20.2/CockroachDB_Training_Wide.png b/images/v20.2/CockroachDB_Training_Wide.png new file mode 100644 index 00000000000..0844c2b50e0 Binary files /dev/null and b/images/v20.2/CockroachDB_Training_Wide.png differ diff --git a/images/v20.2/Parallel_Statement_Execution_Error_Mismatch.png b/images/v20.2/Parallel_Statement_Execution_Error_Mismatch.png new file mode 100644 index 00000000000..f60360c9598 Binary files /dev/null and b/images/v20.2/Parallel_Statement_Execution_Error_Mismatch.png differ diff --git a/images/v20.2/Parallel_Statement_Hybrid_Execution.png b/images/v20.2/Parallel_Statement_Hybrid_Execution.png new file mode 100644 index 00000000000..a4edf85dc02 Binary files /dev/null and b/images/v20.2/Parallel_Statement_Hybrid_Execution.png differ diff --git a/images/v20.2/Parallel_Statement_Normal_Execution.png b/images/v20.2/Parallel_Statement_Normal_Execution.png new file mode 100644 index 00000000000..df63ab1da01 Binary files /dev/null and b/images/v20.2/Parallel_Statement_Normal_Execution.png differ diff --git a/images/v20.2/Sequential_Statement_Execution.png b/images/v20.2/Sequential_Statement_Execution.png new file mode 100644 index 00000000000..99c47c51664 Binary files /dev/null and b/images/v20.2/Sequential_Statement_Execution.png differ diff --git a/images/v20.2/admin-ui-cluster-overview-panel.png b/images/v20.2/admin-ui-cluster-overview-panel.png new file mode 100644 index 00000000000..3d1463f88b7 Binary files /dev/null and b/images/v20.2/admin-ui-cluster-overview-panel.png differ diff --git a/images/v20.2/admin-ui-custom-chart-debug-00.png b/images/v20.2/admin-ui-custom-chart-debug-00.png new file mode 100644 index 00000000000..a82305beffd Binary files /dev/null and b/images/v20.2/admin-ui-custom-chart-debug-00.png differ diff --git a/images/v20.2/admin-ui-custom-chart-debug-01.png b/images/v20.2/admin-ui-custom-chart-debug-01.png new file mode 100644 index 00000000000..f8b9162f14e Binary files /dev/null and b/images/v20.2/admin-ui-custom-chart-debug-01.png differ diff --git a/images/v20.2/admin-ui-decommissioned-nodes.png b/images/v20.2/admin-ui-decommissioned-nodes.png new file mode 100644 index 00000000000..ecd4a1788aa Binary files /dev/null and b/images/v20.2/admin-ui-decommissioned-nodes.png differ diff --git a/images/v20.2/admin-ui-node-components.png b/images/v20.2/admin-ui-node-components.png new file mode 100644 index 00000000000..810a14d246f Binary files /dev/null and b/images/v20.2/admin-ui-node-components.png differ diff --git a/images/v20.2/admin-ui-node-list.png b/images/v20.2/admin-ui-node-list.png new file mode 100644 index 00000000000..5467fcd43ec Binary files /dev/null and b/images/v20.2/admin-ui-node-list.png differ diff --git a/images/v20.2/admin-ui-node-map-after-license.png b/images/v20.2/admin-ui-node-map-after-license.png new file mode 100644 index 00000000000..fa47a7b579f Binary files /dev/null and b/images/v20.2/admin-ui-node-map-after-license.png differ diff --git a/images/v20.2/admin-ui-node-map-before-license.png b/images/v20.2/admin-ui-node-map-before-license.png new file mode 100644 index 00000000000..f352e214868 Binary files /dev/null and b/images/v20.2/admin-ui-node-map-before-license.png differ diff --git a/images/v20.2/admin-ui-node-map-complete.png b/images/v20.2/admin-ui-node-map-complete.png new file mode 100644 index 00000000000..46b1c38d4bf Binary files /dev/null and b/images/v20.2/admin-ui-node-map-complete.png differ diff --git a/images/v20.2/admin-ui-node-map-navigation.gif b/images/v20.2/admin-ui-node-map-navigation.gif new file mode 100644 index 00000000000..67ce2dc009c Binary files /dev/null and b/images/v20.2/admin-ui-node-map-navigation.gif differ diff --git a/images/v20.2/admin-ui-node-map.png b/images/v20.2/admin-ui-node-map.png new file mode 100644 index 00000000000..760d0108c49 Binary files /dev/null and b/images/v20.2/admin-ui-node-map.png differ diff --git a/images/v20.2/admin-ui-region-component.png b/images/v20.2/admin-ui-region-component.png new file mode 100644 index 00000000000..7a9bcbd5b58 Binary files /dev/null and b/images/v20.2/admin-ui-region-component.png differ diff --git a/images/v20.2/admin-ui-single-node.gif b/images/v20.2/admin-ui-single-node.gif new file mode 100644 index 00000000000..f60d25b0e2a Binary files /dev/null and b/images/v20.2/admin-ui-single-node.gif differ diff --git a/images/v20.2/admin-ui-statements-page.png b/images/v20.2/admin-ui-statements-page.png new file mode 100644 index 00000000000..36bdb055e3a Binary files /dev/null and b/images/v20.2/admin-ui-statements-page.png differ diff --git a/images/v20.2/admin-ui-time-range.gif b/images/v20.2/admin-ui-time-range.gif new file mode 100644 index 00000000000..f3d5d7ca2ca Binary files /dev/null and b/images/v20.2/admin-ui-time-range.gif differ diff --git a/images/v20.2/admin_ui_available_disk_capacity.png b/images/v20.2/admin_ui_available_disk_capacity.png new file mode 100644 index 00000000000..7ee4c2c5359 Binary files /dev/null and b/images/v20.2/admin_ui_available_disk_capacity.png differ diff --git a/images/v20.2/admin_ui_capacity.png b/images/v20.2/admin_ui_capacity.png new file mode 100644 index 00000000000..1e9085851af Binary files /dev/null and b/images/v20.2/admin_ui_capacity.png differ diff --git a/images/v20.2/admin_ui_changefeed_restarts.png b/images/v20.2/admin_ui_changefeed_restarts.png new file mode 100644 index 00000000000..7989e71fcb6 Binary files /dev/null and b/images/v20.2/admin_ui_changefeed_restarts.png differ diff --git a/images/v20.2/admin_ui_cluster_overview_3_nodes.png b/images/v20.2/admin_ui_cluster_overview_3_nodes.png new file mode 100644 index 00000000000..1f574610d92 Binary files /dev/null and b/images/v20.2/admin_ui_cluster_overview_3_nodes.png differ diff --git a/images/v20.2/admin_ui_cluster_overview_5_nodes.png b/images/v20.2/admin_ui_cluster_overview_5_nodes.png new file mode 100644 index 00000000000..ec205fface0 Binary files /dev/null and b/images/v20.2/admin_ui_cluster_overview_5_nodes.png differ diff --git a/images/v20.2/admin_ui_cpu_percent.png b/images/v20.2/admin_ui_cpu_percent.png new file mode 100644 index 00000000000..dae468b6d6f Binary files /dev/null and b/images/v20.2/admin_ui_cpu_percent.png differ diff --git a/images/v20.2/admin_ui_cpu_time.png b/images/v20.2/admin_ui_cpu_time.png new file mode 100644 index 00000000000..3e81817ca38 Binary files /dev/null and b/images/v20.2/admin_ui_cpu_time.png differ diff --git a/images/v20.2/admin_ui_database_grants_view.png b/images/v20.2/admin_ui_database_grants_view.png new file mode 100644 index 00000000000..c21145da9f9 Binary files /dev/null and b/images/v20.2/admin_ui_database_grants_view.png differ diff --git a/images/v20.2/admin_ui_database_tables_details.png b/images/v20.2/admin_ui_database_tables_details.png new file mode 100644 index 00000000000..ec640e6ebbd Binary files /dev/null and b/images/v20.2/admin_ui_database_tables_details.png differ diff --git a/images/v20.2/admin_ui_database_tables_view.png b/images/v20.2/admin_ui_database_tables_view.png new file mode 100644 index 00000000000..7dd2eba7eba Binary files /dev/null and b/images/v20.2/admin_ui_database_tables_view.png differ diff --git a/images/v20.2/admin_ui_disk_iops.png b/images/v20.2/admin_ui_disk_iops.png new file mode 100644 index 00000000000..f0f553547e3 Binary files /dev/null and b/images/v20.2/admin_ui_disk_iops.png differ diff --git a/images/v20.2/admin_ui_disk_read_bytes.png b/images/v20.2/admin_ui_disk_read_bytes.png new file mode 100644 index 00000000000..15bcb584f55 Binary files /dev/null and b/images/v20.2/admin_ui_disk_read_bytes.png differ diff --git a/images/v20.2/admin_ui_disk_read_ops.png b/images/v20.2/admin_ui_disk_read_ops.png new file mode 100644 index 00000000000..55b356f84ec Binary files /dev/null and b/images/v20.2/admin_ui_disk_read_ops.png differ diff --git a/images/v20.2/admin_ui_disk_read_time.png b/images/v20.2/admin_ui_disk_read_time.png new file mode 100644 index 00000000000..fd340744135 Binary files /dev/null and b/images/v20.2/admin_ui_disk_read_time.png differ diff --git a/images/v20.2/admin_ui_disk_write_bytes.png b/images/v20.2/admin_ui_disk_write_bytes.png new file mode 100644 index 00000000000..e3fd5fccdad Binary files /dev/null and b/images/v20.2/admin_ui_disk_write_bytes.png differ diff --git a/images/v20.2/admin_ui_disk_write_ops.png b/images/v20.2/admin_ui_disk_write_ops.png new file mode 100644 index 00000000000..9e493d69f88 Binary files /dev/null and b/images/v20.2/admin_ui_disk_write_ops.png differ diff --git a/images/v20.2/admin_ui_disk_write_time.png b/images/v20.2/admin_ui_disk_write_time.png new file mode 100644 index 00000000000..3cd023ffd40 Binary files /dev/null and b/images/v20.2/admin_ui_disk_write_time.png differ diff --git a/images/v20.2/admin_ui_events.png b/images/v20.2/admin_ui_events.png new file mode 100644 index 00000000000..648cb07e7e9 Binary files /dev/null and b/images/v20.2/admin_ui_events.png differ diff --git a/images/v20.2/admin_ui_file_descriptors.png b/images/v20.2/admin_ui_file_descriptors.png new file mode 100644 index 00000000000..42187c9878d Binary files /dev/null and b/images/v20.2/admin_ui_file_descriptors.png differ diff --git a/images/v20.2/admin_ui_hovering.gif b/images/v20.2/admin_ui_hovering.gif new file mode 100644 index 00000000000..1795471051f Binary files /dev/null and b/images/v20.2/admin_ui_hovering.gif differ diff --git a/images/v20.2/admin_ui_jobs_page.png b/images/v20.2/admin_ui_jobs_page.png new file mode 100644 index 00000000000..a9f07a785a3 Binary files /dev/null and b/images/v20.2/admin_ui_jobs_page.png differ diff --git a/images/v20.2/admin_ui_jobs_page_details.png b/images/v20.2/admin_ui_jobs_page_details.png new file mode 100644 index 00000000000..f3bacc50e5b Binary files /dev/null and b/images/v20.2/admin_ui_jobs_page_details.png differ diff --git a/images/v20.2/admin_ui_jobs_page_new.png b/images/v20.2/admin_ui_jobs_page_new.png new file mode 100644 index 00000000000..b193014c8a5 Binary files /dev/null and b/images/v20.2/admin_ui_jobs_page_new.png differ diff --git a/images/v20.2/admin_ui_live_bytes.png b/images/v20.2/admin_ui_live_bytes.png new file mode 100644 index 00000000000..98980af23ed Binary files /dev/null and b/images/v20.2/admin_ui_live_bytes.png differ diff --git a/images/v20.2/admin_ui_logical_bytes_per_store.png b/images/v20.2/admin_ui_logical_bytes_per_store.png new file mode 100644 index 00000000000..75ee063d6e0 Binary files /dev/null and b/images/v20.2/admin_ui_logical_bytes_per_store.png differ diff --git a/images/v20.2/admin_ui_max_changefeed.png b/images/v20.2/admin_ui_max_changefeed.png new file mode 100644 index 00000000000..7fb0eba3d8a Binary files /dev/null and b/images/v20.2/admin_ui_max_changefeed.png differ diff --git a/images/v20.2/admin_ui_memory_usage.png b/images/v20.2/admin_ui_memory_usage.png new file mode 100644 index 00000000000..ffc2c515616 Binary files /dev/null and b/images/v20.2/admin_ui_memory_usage.png differ diff --git a/images/v20.2/admin_ui_memory_usage_new.png b/images/v20.2/admin_ui_memory_usage_new.png new file mode 100644 index 00000000000..97ae93e1b8e Binary files /dev/null and b/images/v20.2/admin_ui_memory_usage_new.png differ diff --git a/images/v20.2/admin_ui_network_bytes_received.png b/images/v20.2/admin_ui_network_bytes_received.png new file mode 100644 index 00000000000..e9a274dc793 Binary files /dev/null and b/images/v20.2/admin_ui_network_bytes_received.png differ diff --git a/images/v20.2/admin_ui_network_bytes_sent.png b/images/v20.2/admin_ui_network_bytes_sent.png new file mode 100644 index 00000000000..2eb35a43222 Binary files /dev/null and b/images/v20.2/admin_ui_network_bytes_sent.png differ diff --git a/images/v20.2/admin_ui_network_latency_collapsed_nodes.png b/images/v20.2/admin_ui_network_latency_collapsed_nodes.png new file mode 100644 index 00000000000..5d7c3b3b662 Binary files /dev/null and b/images/v20.2/admin_ui_network_latency_collapsed_nodes.png differ diff --git a/images/v20.2/admin_ui_network_latency_matrix.png b/images/v20.2/admin_ui_network_latency_matrix.png new file mode 100644 index 00000000000..6b322d3f1ee Binary files /dev/null and b/images/v20.2/admin_ui_network_latency_matrix.png differ diff --git a/images/v20.2/admin_ui_node_count.png b/images/v20.2/admin_ui_node_count.png new file mode 100644 index 00000000000..d5c103fc868 Binary files /dev/null and b/images/v20.2/admin_ui_node_count.png differ diff --git a/images/v20.2/admin_ui_nodes_page.png b/images/v20.2/admin_ui_nodes_page.png new file mode 100644 index 00000000000..495ff14eea0 Binary files /dev/null and b/images/v20.2/admin_ui_nodes_page.png differ diff --git a/images/v20.2/admin_ui_overview_dashboard.png b/images/v20.2/admin_ui_overview_dashboard.png new file mode 100644 index 00000000000..c2adcbf0c83 Binary files /dev/null and b/images/v20.2/admin_ui_overview_dashboard.png differ diff --git a/images/v20.2/admin_ui_overview_dashboard_1_suspect.png b/images/v20.2/admin_ui_overview_dashboard_1_suspect.png new file mode 100644 index 00000000000..49da4bdcf08 Binary files /dev/null and b/images/v20.2/admin_ui_overview_dashboard_1_suspect.png differ diff --git a/images/v20.2/admin_ui_overview_dashboard_3_nodes.png b/images/v20.2/admin_ui_overview_dashboard_3_nodes.png new file mode 100644 index 00000000000..50e2a7148f7 Binary files /dev/null and b/images/v20.2/admin_ui_overview_dashboard_3_nodes.png differ diff --git a/images/v20.2/admin_ui_ranges.png b/images/v20.2/admin_ui_ranges.png new file mode 100644 index 00000000000..316186bb4a3 Binary files /dev/null and b/images/v20.2/admin_ui_ranges.png differ diff --git a/images/v20.2/admin_ui_replica_quiescence.png b/images/v20.2/admin_ui_replica_quiescence.png new file mode 100644 index 00000000000..663dbfb097e Binary files /dev/null and b/images/v20.2/admin_ui_replica_quiescence.png differ diff --git a/images/v20.2/admin_ui_replica_snapshots.png b/images/v20.2/admin_ui_replica_snapshots.png new file mode 100644 index 00000000000..177d8f571ba Binary files /dev/null and b/images/v20.2/admin_ui_replica_snapshots.png differ diff --git a/images/v20.2/admin_ui_replicas_migration.png b/images/v20.2/admin_ui_replicas_migration.png new file mode 100644 index 00000000000..6e08c5a3a5b Binary files /dev/null and b/images/v20.2/admin_ui_replicas_migration.png differ diff --git a/images/v20.2/admin_ui_replicas_migration2.png b/images/v20.2/admin_ui_replicas_migration2.png new file mode 100644 index 00000000000..f7183689f20 Binary files /dev/null and b/images/v20.2/admin_ui_replicas_migration2.png differ diff --git a/images/v20.2/admin_ui_replicas_migration3.png b/images/v20.2/admin_ui_replicas_migration3.png new file mode 100644 index 00000000000..b7d9fd39760 Binary files /dev/null and b/images/v20.2/admin_ui_replicas_migration3.png differ diff --git a/images/v20.2/admin_ui_replicas_per_node.png b/images/v20.2/admin_ui_replicas_per_node.png new file mode 100644 index 00000000000..a6a662c6f32 Binary files /dev/null and b/images/v20.2/admin_ui_replicas_per_node.png differ diff --git a/images/v20.2/admin_ui_replicas_per_store.png b/images/v20.2/admin_ui_replicas_per_store.png new file mode 100644 index 00000000000..2036c392fc8 Binary files /dev/null and b/images/v20.2/admin_ui_replicas_per_store.png differ diff --git a/images/v20.2/admin_ui_service_latency_99_percentile.png b/images/v20.2/admin_ui_service_latency_99_percentile.png new file mode 100644 index 00000000000..7e14805d21d Binary files /dev/null and b/images/v20.2/admin_ui_service_latency_99_percentile.png differ diff --git a/images/v20.2/admin_ui_sink_byte_traffic.png b/images/v20.2/admin_ui_sink_byte_traffic.png new file mode 100644 index 00000000000..4bb61c4e83d Binary files /dev/null and b/images/v20.2/admin_ui_sink_byte_traffic.png differ diff --git a/images/v20.2/admin_ui_sink_counts.png b/images/v20.2/admin_ui_sink_counts.png new file mode 100644 index 00000000000..dc8a6690cbf Binary files /dev/null and b/images/v20.2/admin_ui_sink_counts.png differ diff --git a/images/v20.2/admin_ui_sink_timings.png b/images/v20.2/admin_ui_sink_timings.png new file mode 100644 index 00000000000..63f5de2be4a Binary files /dev/null and b/images/v20.2/admin_ui_sink_timings.png differ diff --git a/images/v20.2/admin_ui_sql_byte_traffic.png b/images/v20.2/admin_ui_sql_byte_traffic.png new file mode 100644 index 00000000000..9f077b25259 Binary files /dev/null and b/images/v20.2/admin_ui_sql_byte_traffic.png differ diff --git a/images/v20.2/admin_ui_sql_connections.png b/images/v20.2/admin_ui_sql_connections.png new file mode 100644 index 00000000000..7cda5614e49 Binary files /dev/null and b/images/v20.2/admin_ui_sql_connections.png differ diff --git a/images/v20.2/admin_ui_sql_queries.png b/images/v20.2/admin_ui_sql_queries.png new file mode 100644 index 00000000000..771c995256e Binary files /dev/null and b/images/v20.2/admin_ui_sql_queries.png differ diff --git a/images/v20.2/admin_ui_sql_query_errors.png b/images/v20.2/admin_ui_sql_query_errors.png new file mode 100644 index 00000000000..6dfe71291f3 Binary files /dev/null and b/images/v20.2/admin_ui_sql_query_errors.png differ diff --git a/images/v20.2/admin_ui_statements_details_page.png b/images/v20.2/admin_ui_statements_details_page.png new file mode 100644 index 00000000000..d94afb43c20 Binary files /dev/null and b/images/v20.2/admin_ui_statements_details_page.png differ diff --git a/images/v20.2/admin_ui_statements_diagnostics.png b/images/v20.2/admin_ui_statements_diagnostics.png new file mode 100644 index 00000000000..bc250924dcf Binary files /dev/null and b/images/v20.2/admin_ui_statements_diagnostics.png differ diff --git a/images/v20.2/admin_ui_statements_logical_plan.png b/images/v20.2/admin_ui_statements_logical_plan.png new file mode 100644 index 00000000000..b6072ce3542 Binary files /dev/null and b/images/v20.2/admin_ui_statements_logical_plan.png differ diff --git a/images/v20.2/admin_ui_summary_panel.png b/images/v20.2/admin_ui_summary_panel.png new file mode 100644 index 00000000000..aabeae3b1df Binary files /dev/null and b/images/v20.2/admin_ui_summary_panel.png differ diff --git a/images/v20.2/admin_ui_transaction_latency.png b/images/v20.2/admin_ui_transaction_latency.png new file mode 100644 index 00000000000..71db3630470 Binary files /dev/null and b/images/v20.2/admin_ui_transaction_latency.png differ diff --git a/images/v20.2/admin_ui_transactions.png b/images/v20.2/admin_ui_transactions.png new file mode 100644 index 00000000000..5131ecc6b2d Binary files /dev/null and b/images/v20.2/admin_ui_transactions.png differ diff --git a/images/v20.2/after-decommission1.png b/images/v20.2/after-decommission1.png new file mode 100644 index 00000000000..945ec05f974 Binary files /dev/null and b/images/v20.2/after-decommission1.png differ diff --git a/images/v20.2/after-decommission2.png b/images/v20.2/after-decommission2.png new file mode 100644 index 00000000000..fbb041d2c14 Binary files /dev/null and b/images/v20.2/after-decommission2.png differ diff --git a/images/v20.2/automated-operations1.png b/images/v20.2/automated-operations1.png new file mode 100644 index 00000000000..64c6e51616c Binary files /dev/null and b/images/v20.2/automated-operations1.png differ diff --git a/images/v20.2/before-decommission1.png b/images/v20.2/before-decommission1.png new file mode 100644 index 00000000000..91627545b22 Binary files /dev/null and b/images/v20.2/before-decommission1.png differ diff --git a/images/v20.2/before-decommission2.png b/images/v20.2/before-decommission2.png new file mode 100644 index 00000000000..063efeb6326 Binary files /dev/null and b/images/v20.2/before-decommission2.png differ diff --git a/images/v20.2/cloudformation_admin_ui_live_node_count.png b/images/v20.2/cloudformation_admin_ui_live_node_count.png new file mode 100644 index 00000000000..fce52a39034 Binary files /dev/null and b/images/v20.2/cloudformation_admin_ui_live_node_count.png differ diff --git a/images/v20.2/cloudformation_admin_ui_replicas.png b/images/v20.2/cloudformation_admin_ui_replicas.png new file mode 100644 index 00000000000..9327b1004e4 Binary files /dev/null and b/images/v20.2/cloudformation_admin_ui_replicas.png differ diff --git a/images/v20.2/cloudformation_admin_ui_sql_queries.png b/images/v20.2/cloudformation_admin_ui_sql_queries.png new file mode 100644 index 00000000000..843d94b30f0 Binary files /dev/null and b/images/v20.2/cloudformation_admin_ui_sql_queries.png differ diff --git a/images/v20.2/cluster-status-after-decommission1.png b/images/v20.2/cluster-status-after-decommission1.png new file mode 100644 index 00000000000..35d96fef0d5 Binary files /dev/null and b/images/v20.2/cluster-status-after-decommission1.png differ diff --git a/images/v20.2/cluster-status-after-decommission2.png b/images/v20.2/cluster-status-after-decommission2.png new file mode 100644 index 00000000000..e420e202aa6 Binary files /dev/null and b/images/v20.2/cluster-status-after-decommission2.png differ diff --git a/images/v20.2/cockroachcloud/access-tab.png b/images/v20.2/cockroachcloud/access-tab.png new file mode 100644 index 00000000000..e8930383d77 Binary files /dev/null and b/images/v20.2/cockroachcloud/access-tab.png differ diff --git a/images/v20.2/cockroachcloud/add-network-modal.png b/images/v20.2/cockroachcloud/add-network-modal.png new file mode 100644 index 00000000000..efb3dd01dad Binary files /dev/null and b/images/v20.2/cockroachcloud/add-network-modal.png differ diff --git a/images/v20.2/cockroachcloud/add-user-modal.png b/images/v20.2/cockroachcloud/add-user-modal.png new file mode 100644 index 00000000000..fa08d017471 Binary files /dev/null and b/images/v20.2/cockroachcloud/add-user-modal.png differ diff --git a/images/v20.2/cockroachcloud/all-clusters-page.png b/images/v20.2/cockroachcloud/all-clusters-page.png new file mode 100644 index 00000000000..6d17da15e27 Binary files /dev/null and b/images/v20.2/cockroachcloud/all-clusters-page.png differ diff --git a/images/v20.2/cockroachcloud/cluster-details-page.png b/images/v20.2/cockroachcloud/cluster-details-page.png new file mode 100644 index 00000000000..108efa6fd15 Binary files /dev/null and b/images/v20.2/cockroachcloud/cluster-details-page.png differ diff --git a/images/v20.2/cockroachcloud/connect-from-app.png b/images/v20.2/cockroachcloud/connect-from-app.png new file mode 100644 index 00000000000..96a41f74803 Binary files /dev/null and b/images/v20.2/cockroachcloud/connect-from-app.png differ diff --git a/images/v20.2/cockroachcloud/connect-modal.png b/images/v20.2/cockroachcloud/connect-modal.png new file mode 100644 index 00000000000..7c99e9d09a2 Binary files /dev/null and b/images/v20.2/cockroachcloud/connect-modal.png differ diff --git a/images/v20.2/cockroachcloud/invite-team-members-modal.png b/images/v20.2/cockroachcloud/invite-team-members-modal.png new file mode 100644 index 00000000000..a4931172754 Binary files /dev/null and b/images/v20.2/cockroachcloud/invite-team-members-modal.png differ diff --git a/images/v20.2/cockroachcloud/networking.png b/images/v20.2/cockroachcloud/networking.png new file mode 100644 index 00000000000..96e2c20e253 Binary files /dev/null and b/images/v20.2/cockroachcloud/networking.png differ diff --git a/images/v20.2/cockroachcloud/sign-up.png b/images/v20.2/cockroachcloud/sign-up.png new file mode 100644 index 00000000000..2e532f9c2d3 Binary files /dev/null and b/images/v20.2/cockroachcloud/sign-up.png differ diff --git a/images/v20.2/cockroachcloud/sql-users.png b/images/v20.2/cockroachcloud/sql-users.png new file mode 100644 index 00000000000..93caeb705e0 Binary files /dev/null and b/images/v20.2/cockroachcloud/sql-users.png differ diff --git a/images/v20.2/concurrency.png b/images/v20.2/concurrency.png new file mode 100644 index 00000000000..e69de29bb2d diff --git a/images/v20.2/dbeaver-01-select-cockroachdb.png b/images/v20.2/dbeaver-01-select-cockroachdb.png new file mode 100644 index 00000000000..d58344cd09c Binary files /dev/null and b/images/v20.2/dbeaver-01-select-cockroachdb.png differ diff --git a/images/v20.2/dbeaver-02-cockroachdb-connection-settings.png b/images/v20.2/dbeaver-02-cockroachdb-connection-settings.png new file mode 100644 index 00000000000..ffd14ff855f Binary files /dev/null and b/images/v20.2/dbeaver-02-cockroachdb-connection-settings.png differ diff --git a/images/v20.2/dbeaver-03-ssl-tab.png b/images/v20.2/dbeaver-03-ssl-tab.png new file mode 100644 index 00000000000..eee8393ec2b Binary files /dev/null and b/images/v20.2/dbeaver-03-ssl-tab.png differ diff --git a/images/v20.2/dbeaver-04-connection-success-dialog.png b/images/v20.2/dbeaver-04-connection-success-dialog.png new file mode 100644 index 00000000000..ce5f157ae27 Binary files /dev/null and b/images/v20.2/dbeaver-04-connection-success-dialog.png differ diff --git a/images/v20.2/dbeaver-05-movr.png b/images/v20.2/dbeaver-05-movr.png new file mode 100644 index 00000000000..339b3003e54 Binary files /dev/null and b/images/v20.2/dbeaver-05-movr.png differ diff --git a/images/v20.2/decommission-multiple1.png b/images/v20.2/decommission-multiple1.png new file mode 100644 index 00000000000..30c90280f7c Binary files /dev/null and b/images/v20.2/decommission-multiple1.png differ diff --git a/images/v20.2/decommission-multiple2.png b/images/v20.2/decommission-multiple2.png new file mode 100644 index 00000000000..d93abcd4acb Binary files /dev/null and b/images/v20.2/decommission-multiple2.png differ diff --git a/images/v20.2/decommission-multiple3.png b/images/v20.2/decommission-multiple3.png new file mode 100644 index 00000000000..3a1d17176de Binary files /dev/null and b/images/v20.2/decommission-multiple3.png differ diff --git a/images/v20.2/decommission-multiple4.png b/images/v20.2/decommission-multiple4.png new file mode 100644 index 00000000000..854c4ba50c9 Binary files /dev/null and b/images/v20.2/decommission-multiple4.png differ diff --git a/images/v20.2/decommission-multiple5.png b/images/v20.2/decommission-multiple5.png new file mode 100644 index 00000000000..3a8621e956b Binary files /dev/null and b/images/v20.2/decommission-multiple5.png differ diff --git a/images/v20.2/decommission-multiple6.png b/images/v20.2/decommission-multiple6.png new file mode 100644 index 00000000000..168ba907be1 Binary files /dev/null and b/images/v20.2/decommission-multiple6.png differ diff --git a/images/v20.2/decommission-multiple7.png b/images/v20.2/decommission-multiple7.png new file mode 100644 index 00000000000..a52d034cf9a Binary files /dev/null and b/images/v20.2/decommission-multiple7.png differ diff --git a/images/v20.2/decommission-scenario1.1.png b/images/v20.2/decommission-scenario1.1.png new file mode 100644 index 00000000000..a66389270de Binary files /dev/null and b/images/v20.2/decommission-scenario1.1.png differ diff --git a/images/v20.2/decommission-scenario1.2.png b/images/v20.2/decommission-scenario1.2.png new file mode 100644 index 00000000000..9b33855e101 Binary files /dev/null and b/images/v20.2/decommission-scenario1.2.png differ diff --git a/images/v20.2/decommission-scenario1.3.png b/images/v20.2/decommission-scenario1.3.png new file mode 100644 index 00000000000..4c1175d956b Binary files /dev/null and b/images/v20.2/decommission-scenario1.3.png differ diff --git a/images/v20.2/decommission-scenario2.1.png b/images/v20.2/decommission-scenario2.1.png new file mode 100644 index 00000000000..2fa8790c556 Binary files /dev/null and b/images/v20.2/decommission-scenario2.1.png differ diff --git a/images/v20.2/decommission-scenario2.2.png b/images/v20.2/decommission-scenario2.2.png new file mode 100644 index 00000000000..391b8e24c0f Binary files /dev/null and b/images/v20.2/decommission-scenario2.2.png differ diff --git a/images/v20.2/decommission-scenario3.1.png b/images/v20.2/decommission-scenario3.1.png new file mode 100644 index 00000000000..db682df3d78 Binary files /dev/null and b/images/v20.2/decommission-scenario3.1.png differ diff --git a/images/v20.2/decommission-scenario3.2.png b/images/v20.2/decommission-scenario3.2.png new file mode 100644 index 00000000000..3571bd0b83e Binary files /dev/null and b/images/v20.2/decommission-scenario3.2.png differ diff --git a/images/v20.2/decommission-scenario3.3.png b/images/v20.2/decommission-scenario3.3.png new file mode 100644 index 00000000000..45f61d9bd18 Binary files /dev/null and b/images/v20.2/decommission-scenario3.3.png differ diff --git a/images/v20.2/explain-analyze-distsql-plan.png b/images/v20.2/explain-analyze-distsql-plan.png new file mode 100644 index 00000000000..d0f0371520c Binary files /dev/null and b/images/v20.2/explain-analyze-distsql-plan.png differ diff --git a/images/v20.2/explain-distsql-plan.png b/images/v20.2/explain-distsql-plan.png new file mode 100644 index 00000000000..946628a88a6 Binary files /dev/null and b/images/v20.2/explain-distsql-plan.png differ diff --git a/images/v20.2/explain-distsql-types-plan.png b/images/v20.2/explain-distsql-types-plan.png new file mode 100644 index 00000000000..685609a6167 Binary files /dev/null and b/images/v20.2/explain-distsql-types-plan.png differ diff --git a/images/v20.2/fault-tolerance-1.png b/images/v20.2/fault-tolerance-1.png new file mode 100644 index 00000000000..6014d6ec7dd Binary files /dev/null and b/images/v20.2/fault-tolerance-1.png differ diff --git a/images/v20.2/fault-tolerance-2.png b/images/v20.2/fault-tolerance-2.png new file mode 100644 index 00000000000..5833a575b22 Binary files /dev/null and b/images/v20.2/fault-tolerance-2.png differ diff --git a/images/v20.2/fault-tolerance-3.png b/images/v20.2/fault-tolerance-3.png new file mode 100644 index 00000000000..8d1166a7068 Binary files /dev/null and b/images/v20.2/fault-tolerance-3.png differ diff --git a/images/v20.2/fault-tolerance-4.png b/images/v20.2/fault-tolerance-4.png new file mode 100644 index 00000000000..124e9bd2a12 Binary files /dev/null and b/images/v20.2/fault-tolerance-4.png differ diff --git a/images/v20.2/fault-tolerance-5.png b/images/v20.2/fault-tolerance-5.png new file mode 100644 index 00000000000..6a9187d97d7 Binary files /dev/null and b/images/v20.2/fault-tolerance-5.png differ diff --git a/images/v20.2/fault-tolerance-6.png b/images/v20.2/fault-tolerance-6.png new file mode 100644 index 00000000000..9ede2c60db4 Binary files /dev/null and b/images/v20.2/fault-tolerance-6.png differ diff --git a/images/v20.2/fault-tolerance-7.png b/images/v20.2/fault-tolerance-7.png new file mode 100644 index 00000000000..797a25c9b0b Binary files /dev/null and b/images/v20.2/fault-tolerance-7.png differ diff --git a/images/v20.2/fault-tolerance-8.png b/images/v20.2/fault-tolerance-8.png new file mode 100644 index 00000000000..e0851fb72fd Binary files /dev/null and b/images/v20.2/fault-tolerance-8.png differ diff --git a/images/v20.2/fault-tolerance-9.png b/images/v20.2/fault-tolerance-9.png new file mode 100644 index 00000000000..537d577cb55 Binary files /dev/null and b/images/v20.2/fault-tolerance-9.png differ diff --git a/images/v20.2/follow-workload-1.png b/images/v20.2/follow-workload-1.png new file mode 100644 index 00000000000..a58fcb2e5ed Binary files /dev/null and b/images/v20.2/follow-workload-1.png differ diff --git a/images/v20.2/follow-workload-2.png b/images/v20.2/follow-workload-2.png new file mode 100644 index 00000000000..47d83c5d4d6 Binary files /dev/null and b/images/v20.2/follow-workload-2.png differ diff --git a/images/v20.2/follow-workload-network-latency.png b/images/v20.2/follow-workload-network-latency.png new file mode 100644 index 00000000000..a3669e56660 Binary files /dev/null and b/images/v20.2/follow-workload-network-latency.png differ diff --git a/images/v20.2/geo-partitioning-cluster-topology.png b/images/v20.2/geo-partitioning-cluster-topology.png new file mode 100644 index 00000000000..ec4ce6d5416 Binary files /dev/null and b/images/v20.2/geo-partitioning-cluster-topology.png differ diff --git a/images/v20.2/geo-partitioning-network-latency.png b/images/v20.2/geo-partitioning-network-latency.png new file mode 100644 index 00000000000..582f0317311 Binary files /dev/null and b/images/v20.2/geo-partitioning-network-latency.png differ diff --git a/images/v20.2/geo-partitioning-node-map-1.png b/images/v20.2/geo-partitioning-node-map-1.png new file mode 100644 index 00000000000..a93d49dc118 Binary files /dev/null and b/images/v20.2/geo-partitioning-node-map-1.png differ diff --git a/images/v20.2/geo-partitioning-node-map-2.png b/images/v20.2/geo-partitioning-node-map-2.png new file mode 100644 index 00000000000..9600dcd7b59 Binary files /dev/null and b/images/v20.2/geo-partitioning-node-map-2.png differ diff --git a/images/v20.2/geo-partitioning-node-map-3.png b/images/v20.2/geo-partitioning-node-map-3.png new file mode 100644 index 00000000000..579959153d4 Binary files /dev/null and b/images/v20.2/geo-partitioning-node-map-3.png differ diff --git a/images/v20.2/geo-partitioning-resiliency-1.png b/images/v20.2/geo-partitioning-resiliency-1.png new file mode 100644 index 00000000000..08c7407af90 Binary files /dev/null and b/images/v20.2/geo-partitioning-resiliency-1.png differ diff --git a/images/v20.2/geo-partitioning-resiliency-2.png b/images/v20.2/geo-partitioning-resiliency-2.png new file mode 100644 index 00000000000..4217566ae57 Binary files /dev/null and b/images/v20.2/geo-partitioning-resiliency-2.png differ diff --git a/images/v20.2/geo-partitioning-sql-latency-after-1.png b/images/v20.2/geo-partitioning-sql-latency-after-1.png new file mode 100644 index 00000000000..b3b0ec029e0 Binary files /dev/null and b/images/v20.2/geo-partitioning-sql-latency-after-1.png differ diff --git a/images/v20.2/geo-partitioning-sql-latency-after-2.png b/images/v20.2/geo-partitioning-sql-latency-after-2.png new file mode 100644 index 00000000000..9d5a27d0ff2 Binary files /dev/null and b/images/v20.2/geo-partitioning-sql-latency-after-2.png differ diff --git a/images/v20.2/geo-partitioning-sql-latency-after-3.png b/images/v20.2/geo-partitioning-sql-latency-after-3.png new file mode 100644 index 00000000000..7c6b0dfae2d Binary files /dev/null and b/images/v20.2/geo-partitioning-sql-latency-after-3.png differ diff --git a/images/v20.2/geo-partitioning-sql-latency-before.png b/images/v20.2/geo-partitioning-sql-latency-before.png new file mode 100644 index 00000000000..5703297a7b1 Binary files /dev/null and b/images/v20.2/geo-partitioning-sql-latency-before.png differ diff --git a/images/v20.2/icon_info.svg b/images/v20.2/icon_info.svg new file mode 100644 index 00000000000..57aac994733 --- /dev/null +++ b/images/v20.2/icon_info.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/images/v20.2/kubernetes-alertmanager-home.png b/images/v20.2/kubernetes-alertmanager-home.png new file mode 100644 index 00000000000..8d1272b27a6 Binary files /dev/null and b/images/v20.2/kubernetes-alertmanager-home.png differ diff --git a/images/v20.2/kubernetes-prometheus-alertmanagers.png b/images/v20.2/kubernetes-prometheus-alertmanagers.png new file mode 100644 index 00000000000..c6ee9f3db79 Binary files /dev/null and b/images/v20.2/kubernetes-prometheus-alertmanagers.png differ diff --git a/images/v20.2/kubernetes-prometheus-alertrules.png b/images/v20.2/kubernetes-prometheus-alertrules.png new file mode 100644 index 00000000000..edb19e06254 Binary files /dev/null and b/images/v20.2/kubernetes-prometheus-alertrules.png differ diff --git a/images/v20.2/kubernetes-prometheus-alerts.png b/images/v20.2/kubernetes-prometheus-alerts.png new file mode 100644 index 00000000000..ed0e89aac8d Binary files /dev/null and b/images/v20.2/kubernetes-prometheus-alerts.png differ diff --git a/images/v20.2/kubernetes-prometheus-graph.png b/images/v20.2/kubernetes-prometheus-graph.png new file mode 100644 index 00000000000..9822717cefc Binary files /dev/null and b/images/v20.2/kubernetes-prometheus-graph.png differ diff --git a/images/v20.2/kubernetes-prometheus-targets.png b/images/v20.2/kubernetes-prometheus-targets.png new file mode 100644 index 00000000000..5e4b917eeb8 Binary files /dev/null and b/images/v20.2/kubernetes-prometheus-targets.png differ diff --git a/images/v20.2/kubernetes-upgrade.png b/images/v20.2/kubernetes-upgrade.png new file mode 100644 index 00000000000..497559cef73 Binary files /dev/null and b/images/v20.2/kubernetes-upgrade.png differ diff --git a/images/v20.2/linearscale.png b/images/v20.2/linearscale.png new file mode 100644 index 00000000000..9843a713461 Binary files /dev/null and b/images/v20.2/linearscale.png differ diff --git a/images/v20.2/movr-schema.png b/images/v20.2/movr-schema.png new file mode 100644 index 00000000000..6f50ec28958 Binary files /dev/null and b/images/v20.2/movr-schema.png differ diff --git a/images/v20.2/movr_v2.png b/images/v20.2/movr_v2.png new file mode 100644 index 00000000000..b9cd96bbcf3 Binary files /dev/null and b/images/v20.2/movr_v2.png differ diff --git a/images/v20.2/parallel-commits-00.png b/images/v20.2/parallel-commits-00.png new file mode 100644 index 00000000000..29d3573db87 Binary files /dev/null and b/images/v20.2/parallel-commits-00.png differ diff --git a/images/v20.2/parallel-commits-01.png b/images/v20.2/parallel-commits-01.png new file mode 100644 index 00000000000..c58603cc36c Binary files /dev/null and b/images/v20.2/parallel-commits-01.png differ diff --git a/images/v20.2/parallel-commits-02.png b/images/v20.2/parallel-commits-02.png new file mode 100644 index 00000000000..a7198424b84 Binary files /dev/null and b/images/v20.2/parallel-commits-02.png differ diff --git a/images/v20.2/parallel-commits-03.png b/images/v20.2/parallel-commits-03.png new file mode 100644 index 00000000000..d1f0b6ab0f9 Binary files /dev/null and b/images/v20.2/parallel-commits-03.png differ diff --git a/images/v20.2/parallel-commits-04.png b/images/v20.2/parallel-commits-04.png new file mode 100644 index 00000000000..2a518a1777e Binary files /dev/null and b/images/v20.2/parallel-commits-04.png differ diff --git a/images/v20.2/parallel-commits-05.png b/images/v20.2/parallel-commits-05.png new file mode 100644 index 00000000000..777c36b9614 Binary files /dev/null and b/images/v20.2/parallel-commits-05.png differ diff --git a/images/v20.2/perf_tuning_concepts1.png b/images/v20.2/perf_tuning_concepts1.png new file mode 100644 index 00000000000..3a086a41c26 Binary files /dev/null and b/images/v20.2/perf_tuning_concepts1.png differ diff --git a/images/v20.2/perf_tuning_concepts2.png b/images/v20.2/perf_tuning_concepts2.png new file mode 100644 index 00000000000..d67b8f253f8 Binary files /dev/null and b/images/v20.2/perf_tuning_concepts2.png differ diff --git a/images/v20.2/perf_tuning_concepts3.png b/images/v20.2/perf_tuning_concepts3.png new file mode 100644 index 00000000000..46d666be55d Binary files /dev/null and b/images/v20.2/perf_tuning_concepts3.png differ diff --git a/images/v20.2/perf_tuning_concepts4.png b/images/v20.2/perf_tuning_concepts4.png new file mode 100644 index 00000000000..b60b19e01bf Binary files /dev/null and b/images/v20.2/perf_tuning_concepts4.png differ diff --git a/images/v20.2/perf_tuning_movr_schema.png b/images/v20.2/perf_tuning_movr_schema.png new file mode 100644 index 00000000000..262adc18b75 Binary files /dev/null and b/images/v20.2/perf_tuning_movr_schema.png differ diff --git a/images/v20.2/perf_tuning_multi_region_rebalancing.png b/images/v20.2/perf_tuning_multi_region_rebalancing.png new file mode 100644 index 00000000000..7064e3962db Binary files /dev/null and b/images/v20.2/perf_tuning_multi_region_rebalancing.png differ diff --git a/images/v20.2/perf_tuning_multi_region_rebalancing_after_partitioning.png b/images/v20.2/perf_tuning_multi_region_rebalancing_after_partitioning.png new file mode 100644 index 00000000000..433c0f8ba03 Binary files /dev/null and b/images/v20.2/perf_tuning_multi_region_rebalancing_after_partitioning.png differ diff --git a/images/v20.2/perf_tuning_multi_region_topology.png b/images/v20.2/perf_tuning_multi_region_topology.png new file mode 100644 index 00000000000..fe64c322ca0 Binary files /dev/null and b/images/v20.2/perf_tuning_multi_region_topology.png differ diff --git a/images/v20.2/perf_tuning_single_region_topology.png b/images/v20.2/perf_tuning_single_region_topology.png new file mode 100644 index 00000000000..4dfca364929 Binary files /dev/null and b/images/v20.2/perf_tuning_single_region_topology.png differ diff --git a/images/v20.2/raw-status-endpoints.png b/images/v20.2/raw-status-endpoints.png new file mode 100644 index 00000000000..a893911fa87 Binary files /dev/null and b/images/v20.2/raw-status-endpoints.png differ diff --git a/images/v20.2/recovery1.png b/images/v20.2/recovery1.png new file mode 100644 index 00000000000..8a14f7e965a Binary files /dev/null and b/images/v20.2/recovery1.png differ diff --git a/images/v20.2/recovery2.png b/images/v20.2/recovery2.png new file mode 100644 index 00000000000..7ec3fed2adc Binary files /dev/null and b/images/v20.2/recovery2.png differ diff --git a/images/v20.2/recovery3.png b/images/v20.2/recovery3.png new file mode 100644 index 00000000000..a82da79f64a Binary files /dev/null and b/images/v20.2/recovery3.png differ diff --git a/images/v20.2/remove-dead-node1.png b/images/v20.2/remove-dead-node1.png new file mode 100644 index 00000000000..26569078efd Binary files /dev/null and b/images/v20.2/remove-dead-node1.png differ diff --git a/images/v20.2/replication1.png b/images/v20.2/replication1.png new file mode 100644 index 00000000000..45a4c5dc216 Binary files /dev/null and b/images/v20.2/replication1.png differ diff --git a/images/v20.2/replication2.png b/images/v20.2/replication2.png new file mode 100644 index 00000000000..99f9a09e61a Binary files /dev/null and b/images/v20.2/replication2.png differ diff --git a/images/v20.2/scalability1.png b/images/v20.2/scalability1.png new file mode 100644 index 00000000000..7a70afb6d6a Binary files /dev/null and b/images/v20.2/scalability1.png differ diff --git a/images/v20.2/scalability2.png b/images/v20.2/scalability2.png new file mode 100644 index 00000000000..400748466c6 Binary files /dev/null and b/images/v20.2/scalability2.png differ diff --git a/images/v20.2/serializable_schema.png b/images/v20.2/serializable_schema.png new file mode 100644 index 00000000000..7e8b4e324c6 Binary files /dev/null and b/images/v20.2/serializable_schema.png differ diff --git a/images/v20.2/sysbench-latency.png b/images/v20.2/sysbench-latency.png new file mode 100644 index 00000000000..325f5fb2cff Binary files /dev/null and b/images/v20.2/sysbench-latency.png differ diff --git a/images/v20.2/sysbench-throughput.png b/images/v20.2/sysbench-throughput.png new file mode 100644 index 00000000000..ce09d762da7 Binary files /dev/null and b/images/v20.2/sysbench-throughput.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production1.png b/images/v20.2/topology-patterns/topology_basic_production1.png new file mode 100644 index 00000000000..b96d185197b Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production1.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production2.png b/images/v20.2/topology-patterns/topology_basic_production2.png new file mode 100644 index 00000000000..22359506c75 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production2.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production_reads.png b/images/v20.2/topology-patterns/topology_basic_production_reads.png new file mode 100644 index 00000000000..fd6b9a35e40 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production_resiliency1.png b/images/v20.2/topology-patterns/topology_basic_production_resiliency1.png new file mode 100644 index 00000000000..218e3443668 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production_resiliency1.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production_resiliency2.png b/images/v20.2/topology-patterns/topology_basic_production_resiliency2.png new file mode 100644 index 00000000000..a9efce59a67 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production_resiliency2.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production_resiliency3.png b/images/v20.2/topology-patterns/topology_basic_production_resiliency3.png new file mode 100644 index 00000000000..3c3fd57b457 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production_resiliency3.png differ diff --git a/images/v20.2/topology-patterns/topology_basic_production_writes.gif b/images/v20.2/topology-patterns/topology_basic_production_writes.gif new file mode 100644 index 00000000000..5f12f331e7f Binary files /dev/null and b/images/v20.2/topology-patterns/topology_basic_production_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_development1.png b/images/v20.2/topology-patterns/topology_development1.png new file mode 100644 index 00000000000..2882937e438 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_development1.png differ diff --git a/images/v20.2/topology-patterns/topology_development2.png b/images/v20.2/topology-patterns/topology_development2.png new file mode 100644 index 00000000000..1eed95fbaba Binary files /dev/null and b/images/v20.2/topology-patterns/topology_development2.png differ diff --git a/images/v20.2/topology-patterns/topology_development_latency.png b/images/v20.2/topology-patterns/topology_development_latency.png new file mode 100644 index 00000000000..3aa54c45c13 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_development_latency.png differ diff --git a/images/v20.2/topology-patterns/topology_duplicate_indexes1.png b/images/v20.2/topology-patterns/topology_duplicate_indexes1.png new file mode 100644 index 00000000000..c9ad5d97fa3 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_duplicate_indexes1.png differ diff --git a/images/v20.2/topology-patterns/topology_duplicate_indexes_reads.png b/images/v20.2/topology-patterns/topology_duplicate_indexes_reads.png new file mode 100644 index 00000000000..097927ea410 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_duplicate_indexes_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_duplicate_indexes_resiliency.png b/images/v20.2/topology-patterns/topology_duplicate_indexes_resiliency.png new file mode 100644 index 00000000000..39056e22a48 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_duplicate_indexes_resiliency.png differ diff --git a/images/v20.2/topology-patterns/topology_duplicate_indexes_writes.gif b/images/v20.2/topology-patterns/topology_duplicate_indexes_writes.gif new file mode 100644 index 00000000000..16433549cb4 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_duplicate_indexes_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_follow_the_workload_reads.png b/images/v20.2/topology-patterns/topology_follow_the_workload_reads.png new file mode 100644 index 00000000000..67b01da4d37 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follow_the_workload_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_follow_the_workload_writes.gif b/images/v20.2/topology-patterns/topology_follow_the_workload_writes.gif new file mode 100644 index 00000000000..6cd6be01196 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follow_the_workload_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_follower_reads1.png b/images/v20.2/topology-patterns/topology_follower_reads1.png new file mode 100644 index 00000000000..1eb07d53d6a Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follower_reads1.png differ diff --git a/images/v20.2/topology-patterns/topology_follower_reads3.png b/images/v20.2/topology-patterns/topology_follower_reads3.png new file mode 100644 index 00000000000..d6a125c1079 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follower_reads3.png differ diff --git a/images/v20.2/topology-patterns/topology_follower_reads_reads.png b/images/v20.2/topology-patterns/topology_follower_reads_reads.png new file mode 100644 index 00000000000..47657b885b3 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follower_reads_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_follower_reads_resiliency.png b/images/v20.2/topology-patterns/topology_follower_reads_resiliency.png new file mode 100644 index 00000000000..73868163a1e Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follower_reads_resiliency.png differ diff --git a/images/v20.2/topology-patterns/topology_follower_reads_writes.gif b/images/v20.2/topology-patterns/topology_follower_reads_writes.gif new file mode 100644 index 00000000000..8fc4b2c55b7 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_follower_reads_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders1.png b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders1.png new file mode 100644 index 00000000000..66d03a7f113 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders1.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_reads.png b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_reads.png new file mode 100644 index 00000000000..0daa6665d05 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency1.png b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency1.png new file mode 100644 index 00000000000..09aaa95ded9 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency1.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency2.png b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency2.png new file mode 100644 index 00000000000..f372f14552c Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_resiliency2.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_writes.gif b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_writes.gif new file mode 100644 index 00000000000..f5c8d077818 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioned_leaseholders_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning1.png b/images/v20.2/topology-patterns/topology_geo-partitioning1.png new file mode 100644 index 00000000000..a7bc25e6279 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning1.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning1_no-map.png b/images/v20.2/topology-patterns/topology_geo-partitioning1_no-map.png new file mode 100644 index 00000000000..3b348dd7430 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning1_no-map.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning_reads.png b/images/v20.2/topology-patterns/topology_geo-partitioning_reads.png new file mode 100644 index 00000000000..6dcdd7e418e Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning_reads.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency1.png b/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency1.png new file mode 100644 index 00000000000..d3353c2f8d0 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency1.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency2.png b/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency2.png new file mode 100644 index 00000000000..04191e8ddef Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning_resiliency2.png differ diff --git a/images/v20.2/topology-patterns/topology_geo-partitioning_writes.gif b/images/v20.2/topology-patterns/topology_geo-partitioning_writes.gif new file mode 100644 index 00000000000..11435a6bd51 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_geo-partitioning_writes.gif differ diff --git a/images/v20.2/topology-patterns/topology_multi-region_hardware.png b/images/v20.2/topology-patterns/topology_multi-region_hardware.png new file mode 100644 index 00000000000..dad856590d0 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_multi-region_hardware.png differ diff --git a/images/v20.2/topology-patterns/topology_pinned_index_leaseholders3.png b/images/v20.2/topology-patterns/topology_pinned_index_leaseholders3.png new file mode 100644 index 00000000000..7d792d3a5ed Binary files /dev/null and b/images/v20.2/topology-patterns/topology_pinned_index_leaseholders3.png differ diff --git a/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency1.png b/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency1.png new file mode 100644 index 00000000000..7fe13079fe0 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency1.png differ diff --git a/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency2.png b/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency2.png new file mode 100644 index 00000000000..f11c5989677 Binary files /dev/null and b/images/v20.2/topology-patterns/topology_single-region_cluster_resiliency2.png differ diff --git a/images/v20.2/tpcc100k.png b/images/v20.2/tpcc100k.png new file mode 100644 index 00000000000..ab5cd6c532f Binary files /dev/null and b/images/v20.2/tpcc100k.png differ diff --git a/images/v20.2/tpcc50k.png b/images/v20.2/tpcc50k.png new file mode 100644 index 00000000000..c9086f59998 Binary files /dev/null and b/images/v20.2/tpcc50k.png differ diff --git a/images/v20.2/trace.png b/images/v20.2/trace.png new file mode 100644 index 00000000000..4f0fb98a753 Binary files /dev/null and b/images/v20.2/trace.png differ diff --git a/images/v20.2/training-1.1.png b/images/v20.2/training-1.1.png new file mode 100644 index 00000000000..d1adf35bcde Binary files /dev/null and b/images/v20.2/training-1.1.png differ diff --git a/images/v20.2/training-1.2.png b/images/v20.2/training-1.2.png new file mode 100644 index 00000000000..1993355b08e Binary files /dev/null and b/images/v20.2/training-1.2.png differ diff --git a/images/v20.2/training-1.png b/images/v20.2/training-1.png new file mode 100644 index 00000000000..9f8de513337 Binary files /dev/null and b/images/v20.2/training-1.png differ diff --git a/images/v20.2/training-10.png b/images/v20.2/training-10.png new file mode 100644 index 00000000000..b319a5bf490 Binary files /dev/null and b/images/v20.2/training-10.png differ diff --git a/images/v20.2/training-11.png b/images/v20.2/training-11.png new file mode 100644 index 00000000000..2af80764aaf Binary files /dev/null and b/images/v20.2/training-11.png differ diff --git a/images/v20.2/training-12.png b/images/v20.2/training-12.png new file mode 100644 index 00000000000..7a8e4cd8e05 Binary files /dev/null and b/images/v20.2/training-12.png differ diff --git a/images/v20.2/training-13.png b/images/v20.2/training-13.png new file mode 100644 index 00000000000..fc870143136 Binary files /dev/null and b/images/v20.2/training-13.png differ diff --git a/images/v20.2/training-14.png b/images/v20.2/training-14.png new file mode 100644 index 00000000000..fe517518ed7 Binary files /dev/null and b/images/v20.2/training-14.png differ diff --git a/images/v20.2/training-15.png b/images/v20.2/training-15.png new file mode 100644 index 00000000000..1879ee29d2e Binary files /dev/null and b/images/v20.2/training-15.png differ diff --git a/images/v20.2/training-16.png b/images/v20.2/training-16.png new file mode 100644 index 00000000000..24f6fa3d908 Binary files /dev/null and b/images/v20.2/training-16.png differ diff --git a/images/v20.2/training-17.png b/images/v20.2/training-17.png new file mode 100644 index 00000000000..9bb5c8a46dd Binary files /dev/null and b/images/v20.2/training-17.png differ diff --git a/images/v20.2/training-18.png b/images/v20.2/training-18.png new file mode 100644 index 00000000000..8f0ae7aa857 Binary files /dev/null and b/images/v20.2/training-18.png differ diff --git a/images/v20.2/training-19.png b/images/v20.2/training-19.png new file mode 100644 index 00000000000..e1a2414bf29 Binary files /dev/null and b/images/v20.2/training-19.png differ diff --git a/images/v20.2/training-2.png b/images/v20.2/training-2.png new file mode 100644 index 00000000000..d6d8afd7828 Binary files /dev/null and b/images/v20.2/training-2.png differ diff --git a/images/v20.2/training-20.png b/images/v20.2/training-20.png new file mode 100644 index 00000000000..d55c4f249ae Binary files /dev/null and b/images/v20.2/training-20.png differ diff --git a/images/v20.2/training-21.png b/images/v20.2/training-21.png new file mode 100644 index 00000000000..5726c9c69a7 Binary files /dev/null and b/images/v20.2/training-21.png differ diff --git a/images/v20.2/training-22.png b/images/v20.2/training-22.png new file mode 100644 index 00000000000..fe2ca336a95 Binary files /dev/null and b/images/v20.2/training-22.png differ diff --git a/images/v20.2/training-23.png b/images/v20.2/training-23.png new file mode 100644 index 00000000000..de87538279f Binary files /dev/null and b/images/v20.2/training-23.png differ diff --git a/images/v20.2/training-3.png b/images/v20.2/training-3.png new file mode 100644 index 00000000000..02b5724da59 Binary files /dev/null and b/images/v20.2/training-3.png differ diff --git a/images/v20.2/training-4.png b/images/v20.2/training-4.png new file mode 100644 index 00000000000..ae55051e60e Binary files /dev/null and b/images/v20.2/training-4.png differ diff --git a/images/v20.2/training-5.png b/images/v20.2/training-5.png new file mode 100644 index 00000000000..65c805404c4 Binary files /dev/null and b/images/v20.2/training-5.png differ diff --git a/images/v20.2/training-6.1.png b/images/v20.2/training-6.1.png new file mode 100644 index 00000000000..128ab631ce8 Binary files /dev/null and b/images/v20.2/training-6.1.png differ diff --git a/images/v20.2/training-6.png b/images/v20.2/training-6.png new file mode 100644 index 00000000000..8d93f4c3e3d Binary files /dev/null and b/images/v20.2/training-6.png differ diff --git a/images/v20.2/training-7.png b/images/v20.2/training-7.png new file mode 100644 index 00000000000..46179bfd04b Binary files /dev/null and b/images/v20.2/training-7.png differ diff --git a/images/v20.2/training-8.png b/images/v20.2/training-8.png new file mode 100644 index 00000000000..d31f2e95a29 Binary files /dev/null and b/images/v20.2/training-8.png differ diff --git a/images/v20.2/training-9.png b/images/v20.2/training-9.png new file mode 100644 index 00000000000..f386b9a9aa7 Binary files /dev/null and b/images/v20.2/training-9.png differ diff --git a/images/v20.2/window-functions.png b/images/v20.2/window-functions.png new file mode 100644 index 00000000000..887ceeac669 Binary files /dev/null and b/images/v20.2/window-functions.png differ diff --git a/releases/v20.2.0-alpha.1.md b/releases/v20.2.0-alpha.1.md new file mode 100644 index 00000000000..c2746e701ae --- /dev/null +++ b/releases/v20.2.0-alpha.1.md @@ -0,0 +1,50 @@ +--- +title: What's New in v20.2.0-alpha.1 +toc: true +summary: Additions and changes in CockroachDB version v20.2.0-alpha.1 +--- + +## June 2020 + +{{site.data.alerts.callout_info}} +The release notes for v20.2.0-alpha.1 are in progress (see Github issue [cockroachdb/docs#7426](https://github.com/cockroachdb/docs/issues/7426) and will be published soon. +{{site.data.alerts.end}} + +Get future release notes emailed to you: + +
+ +
+ +## Downloads + +
+ + + + +
+ +## Docker image + +{% include copy-clipboard.html %} +~~~ shell +$ docker pull cockroachdb/cockroach-unstable:v20.2.0-alpha.1 +~~~ + +### Backward-incompatible changes + +To be added soon. + +### Known limitations + +To be added soon. diff --git a/v20.2/404.md b/v20.2/404.md new file mode 100644 index 00000000000..13a69ddde5c --- /dev/null +++ b/v20.2/404.md @@ -0,0 +1,19 @@ +--- +title: Page Not Found +description: "Page not found." +sitemap: false +search: exclude +related_pages: none +toc: false +--- + + +{%comment%} + + +{%endcomment%} \ No newline at end of file diff --git a/v20.2/add-column.md b/v20.2/add-column.md new file mode 100644 index 00000000000..5584ce60376 --- /dev/null +++ b/v20.2/add-column.md @@ -0,0 +1,152 @@ +--- +title: ADD COLUMN +summary: Use the ADD COLUMN statement to add columns to tables. +toc: true +--- + +The `ADD COLUMN` [statement](sql-statements.html) is part of `ALTER TABLE` and adds columns to tables. + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
+{% include {{ page.version.version }}/sql/diagrams/add_column.html %} +
+ +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + + Parameter | Description +-----------|------------- + `table_name` | The name of the table to which you want to add the column. + `column_name` | The name of the column you want to add. The column name must follow these [identifier rules](keywords-and-identifiers.html#identifiers) and must be unique within the table but can have the same name as indexes or constraints. + `typename` | The [data type](data-types.html) of the new column. + `col_qualification` | An optional list of column definitions, which may include [column-level constraints](constraints.html), [collation](collate.html), or [column family assignments](column-families.html).

If the column family is not specified, the column will be added to the first column family. For more information about how column families are assigned, see [Column Families](column-families.html#assign-column-families-when-adding-columns).

Note that it is not possible to add a column with the [foreign key](foreign-key.html) constraint. As a workaround, you can add the column without the constraint, then use [`CREATE INDEX`](create-index.html) to index the column, and then use [`ADD CONSTRAINT`](add-constraint.html) to add the foreign key constraint to the column. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Add a single column + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN names STRING; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM accounts; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| id | INT | false | NULL | | {"primary"} | +| balance | DECIMAL | true | NULL | | {} | +| names | STRING | true | NULL | | {} | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +(3 rows) +~~~ + +### Add multiple columns + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN location STRING, ADD COLUMN amount DECIMAL; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM accounts; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| id | INT | false | NULL | | {"primary"} | +| balance | DECIMAL | true | NULL | | {} | +| names | STRING | true | NULL | | {} | +| location | STRING | true | NULL | | {} | +| amount | DECIMAL | true | NULL | | {} | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +(5 rows) +~~~ + +### Add a column with a `NOT NULL` constraint and a `DEFAULT` value + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN interest DECIMAL NOT NULL DEFAULT (DECIMAL '1.3'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM accounts; +~~~ +~~~ ++-------------+-----------+-------------+------------------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+------------------------+-----------------------+-------------+ +| id | INT | false | NULL | | {"primary"} | +| balance | DECIMAL | true | NULL | | {} | +| names | STRING | true | NULL | | {} | +| location | STRING | true | NULL | | {} | +| amount | DECIMAL | true | NULL | | {} | +| interest | DECIMAL | false | 1.3:::DECIMAL::DECIMAL | | {} | ++-------------+-----------+-------------+------------------------+-----------------------+-------------+ +(6 rows) +~~~ + +### Add a column with `NOT NULL` and `UNIQUE` constraints + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN cust_number DECIMAL UNIQUE NOT NULL; +~~~ + +### Add a column with collation + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN more_names STRING COLLATE en; +~~~ + +### Add a column and assign it to a column family + +#### Add a column and assign it to a new column family + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN location1 STRING CREATE FAMILY new_family; +~~~ + +#### Add a column and assign it to an existing column family + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN location2 STRING FAMILY existing_family; +~~~ + +#### Add a column and create a new column family if column family does not exist + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE accounts ADD COLUMN new_name STRING CREATE IF NOT EXISTS FAMILY f1; +~~~ + +## See also + +- [`ALTER TABLE`](alter-table.html) +- [Column-level Constraints](constraints.html) +- [Collation](collate.html) +- [Column Families](column-families.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/add-constraint.md b/v20.2/add-constraint.md new file mode 100644 index 00000000000..420594844fe --- /dev/null +++ b/v20.2/add-constraint.md @@ -0,0 +1,263 @@ +--- +title: ADD CONSTRAINT +summary: Use the ADD CONSTRAINT statement to add constraints to columns. +toc: true +--- + +The `ADD CONSTRAINT` [statement](sql-statements.html) is part of `ALTER TABLE` and can add the following [constraints](constraints.html) to columns: + +- [`UNIQUE`](#add-the-unique-constraint) +- [`CHECK`](#add-the-check-constraint) +- [`FOREIGN KEY`](#add-the-foreign-key-constraint-with-cascade) + +To add a primary key constraint to a table, you should explicitly define the primary key at [table creation](create-table.html). To replace an existing primary key, you can use `ADD CONSTRAINT ... PRIMARY KEY`. For details, see [Changing primary keys with `ADD CONSTRAINT ... PRIMARY KEY`](#changing-primary-keys-with-add-constraint-primary-key). + +The [`DEFAULT`](default-value.html) and [`NOT NULL`](not-null.html) constraints are managed through [`ALTER COLUMN`](alter-column.html). + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
+{% include {{ page.version.version }}/sql/diagrams/add_constraint.html %} +
+ +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + + Parameter | Description +-----------|------------- + `table_name` | The name of the table containing the column you want to constrain. + `constraint_name` | The name of the constraint, which must be unique to its table and follow these [identifier rules](keywords-and-identifiers.html#identifiers). + `constraint_elem` | The [`CHECK`](check.html), [foreign key](foreign-key.html), [`UNIQUE`](unique.html) constraint you want to add.

Adding/changing a `DEFAULT` constraint is done through [`ALTER COLUMN`](alter-column.html).

Adding/changing the table's `PRIMARY KEY` is not supported through `ALTER TABLE`; it can only be specified during [table creation](create-table.html). + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Changing primary keys with `ADD CONSTRAINT ... PRIMARY KEY` + + When you change a primary key with [`ALTER TABLE ... ALTER PRIMARY KEY`](alter-primary-key.html), the old primary key index becomes a secondary index. The secondary index created by `ALTER PRIMARY KEY` takes up node memory and can slow down write performance to a cluster. If you do not have queries that filter on the primary key that you are replacing, you can use `ADD CONSTRAINT` to replace the old primary index without creating a secondary index. + +`ADD CONSTRAINT ... PRIMARY KEY` can be used to add a primary key to an existing table if one of the following is true: + + - No primary key was explicitly defined at [table creation](create-table.html). In this case, the table is created with a default [primary key on `rowid`](indexes.html#creation). Using `ADD CONSTRAINT ... PRIMARY KEY` drops the default primary key and replaces it with a new primary key. + - A [`DROP CONSTRAINT`](drop-constraint.html) statement precedes the `ADD CONSTRAINT ... PRIMARY KEY` statement, in the same transaction. For an example, see [Drop and add the primary key constraint](#drop-and-add-a-primary-key-constraint) below. + +{{site.data.alerts.callout_info}} +`ALTER TABLE ... ADD PRIMARY KEY` is an alias for `ALTER TABLE ... ADD CONSTRAINT ... PRIMARY KEY`. +{{site.data.alerts.end}} + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Add the `UNIQUE` constraint + +Adding the [`UNIQUE` constraint](unique.html) requires that all of a column's values be distinct from one another (except for *NULL* values). + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD CONSTRAINT id_name_unique UNIQUE (id, name); +~~~ + +### Add the `CHECK` constraint + +Adding the [`CHECK` constraint](check.html) requires that all of a column's values evaluate to `TRUE` for a Boolean expression. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE rides ADD CONSTRAINT check_revenue_positive CHECK (revenue >= 0); +~~~ + +Check constraints can be added to columns that were created earlier in the transaction. For example: + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +> ALTER TABLE users ADD COLUMN is_owner STRING; +> ALTER TABLE users ADD CONSTRAINT check_is_owner CHECK (is_owner IN ('yes', 'no', 'unknown')); +> COMMIT; +~~~ + +~~~ +BEGIN +ALTER TABLE +ALTER TABLE +COMMIT +~~~ + +{{site.data.alerts.callout_info}} +The entire transaction will be rolled back, including any new columns that were added, in the following cases: + +- If an existing column is found containing values that violate the new constraint. +- If a new column has a default value or is a [computed column](computed-columns.html) that would have contained values that violate the new constraint. +{{site.data.alerts.end}} + +### Add the foreign key constraint with `CASCADE` + +To add a foreign key constraint, use the steps shown below. + +Given two tables, `users` and `vehicles`, without foreign key constraints: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE users; +~~~ + +~~~ + table_name | create_statement +-------------+-------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE vehicles; +~~~ + +~~~ + table_name | create_statement +-------------+------------------------------------------------------------------------------------------------ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | type VARCHAR NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status VARCHAR NULL, + | current_location VARCHAR NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) +(1 row) +~~~ + +You can include a [foreign key action](foreign-key.html#foreign-key-actions) to specify what happens when a foreign key is updated or deleted. + +Using `ON DELETE CASCADE` will ensure that when the referenced row is deleted, all dependent objects are also deleted. + +{{site.data.alerts.callout_danger}} +`CASCADE` does not list the objects it drops or updates, so it should be used with caution. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE vehicles ADD CONSTRAINT users_fk FOREIGN KEY (city, owner_id) REFERENCES users (city, id) ON DELETE CASCADE; +~~~ + +An index on the referencing columns is automatically created for you when you add a foreign key constraint to an empty table, if an appropriate index does not already exist. You can see it using [`SHOW INDEXES`](show-index.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEXES FROM vehicles; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit +-------------+------------------------------+------------+--------------+-------------+-----------+---------+----------- + vehicles | primary | false | 1 | city | ASC | false | false + vehicles | primary | false | 2 | id | ASC | false | false + vehicles | vehicles_auto_index_users_fk | true | 1 | city | ASC | false | false + vehicles | vehicles_auto_index_users_fk | true | 2 | owner_id | ASC | false | false + vehicles | vehicles_auto_index_users_fk | true | 3 | id | ASC | false | true +(5 rows) +~~~ + +{{site.data.alerts.callout_info}} +Adding a foreign key for a non-empty table without an appropriate index will fail, since foreign key columns must be indexed. For more information about the requirements for creating foreign keys, see [Rules for creating foreign keys](foreign-key.html#rules-for-creating-foreign-keys). +{{site.data.alerts.end}} + +### Drop and add a primary key constraint + +Suppose that you want to add `name` to the composite primary key of the `users` table, [without creating a secondary index of the existing primary key](#changing-primary-keys-with-add-constraint-primary-key). + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+-------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) +(1 row) +~~~ + +First, add a [`NOT NULL`](not-null.html) constraint to the `name` column with [`ALTER COLUMN`](alter-column.html). + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ALTER COLUMN name SET NOT NULL; +~~~ + +Then, in the same transaction, [`DROP`](drop-constraint.html) the old `"primary"` constraint and `ADD` the new one: + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +> ALTER TABLE users DROP CONSTRAINT "primary"; +> ALTER TABLE users ADD CONSTRAINT "primary" PRIMARY KEY (city, name, id); +> COMMIT; +~~~ + +~~~ +NOTICE: primary key changes are finalized asynchronously; further schema changes on this table may be restricted until the job completes +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+--------------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NOT NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, name ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) +(1 row) +~~~ + +Using [`ALTER PRIMARY KEY`](alter-primary-key.html) would have created a `UNIQUE` secondary index called `users_city_id_key`. Instead, there is just one index for the primary key constraint. + +## See also + +- [Constraints](constraints.html) +- [Foreign Key Constraint](foreign-key.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) +- [`RENAME CONSTRAINT`](rename-constraint.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`VALIDATE CONSTRAINT`](validate-constraint.html) +- [`ALTER COLUMN`](alter-column.html) +- [`CREATE TABLE`](create-table.html) +- [`ALTER TABLE`](alter-table.html) +- [`SHOW JOBS`](show-jobs.html) +- ['ALTER PRIMARY KEY'](alter-primary-key.html) diff --git a/v20.2/admin-ui-access-and-navigate.md b/v20.2/admin-ui-access-and-navigate.md new file mode 100644 index 00000000000..8320a042b9d --- /dev/null +++ b/v20.2/admin-ui-access-and-navigate.md @@ -0,0 +1,128 @@ +--- +title: Use the CockroachDB Admin UI +summary: Learn how to access and navigate the Admin UI. +toc: true +--- + +Use the [Admin UI](admin-ui-overview.html) to monitor and troubleshoot CockroachDB by viewing the cluster's health, configuration, and operations. + +## Access the Admin UI + +For insecure clusters, anyone can access the Admin UI. For secure clusters, only authorized users can [access the Admin UI](#accessing-the-admin-ui-for-a-secure-cluster). In addition, certain areas of the Admin UI can only be [accessed by `admin` users](admin-ui-overview.html#admin-ui-access). + +You can access the Admin UI from any node in the cluster. + +The Admin UI is reachable at the IP address/hostname and port set via the `--http-addr` flag when [starting each node](cockroach-start.html). For example, `http://
:` for an insecure cluster or `https://
:` for a secure cluster. + +If `--http-addr` is not specified when starting a node, the Admin UI is reachable at the IP address/hostname set via the `--listen-addr` flag and port `8080`. + +For additional guidance on accessing the Admin UI in the context of cluster deployment, see [Start a Local Cluster](start-a-local-cluster.html) and [Manual Deployment](manual-deployment.html). + +### Accessing the Admin UI for a secure cluster + +Note that on secure clusters, certain areas of the Admin UI can only be accessed by `admin` users. For details on providing access to users, see [this page](admin-ui-overview.html#admin-ui-access). + +On accessing the Admin UI, your browser will consider the CockroachDB-created certificate invalid, so you’ll need to click through a warning message to get to the UI. For secure clusters, you can avoid getting the warning message by using a certificate issued by a public CA. For more information, refer to [Use a UI certificate and key to access the Admin UI](create-security-certificates-custom-ca.html#accessing-the-admin-ui-for-a-secure-cluster). + +For each user who should have access to the Admin UI for a secure cluster, [create a SQL user with a password](create-user.html). On accessing the Admin UI, the users will see a Login screen, where they will need to enter their usernames and passwords. + +{{site.data.alerts.callout_info}} +This login information is stored in a system table that is replicated like other data in the cluster. If a majority of the nodes with the replicas of the system table data go down, users will be locked out of the Admin UI. +{{site.data.alerts.end}} + +To log out of the Admin UI, click the user icon at the top-right corner and then **Logout**. + +## Use the Admin UI + +{{site.data.alerts.callout_success}} +See [Admin UI Overview](admin-ui-overview.html#admin-ui-areas) for a full list of Admin UI areas. +{{site.data.alerts.end}} + +### Metrics + +The Metrics dashboards display timeseries graphs that visualize data and enable you to monitor trends. To access the timeseries graphs, click **Metrics** on the left. + +You can hover over each graph to see actual point-in-time values. + +CockroachDB Admin UI + +{{site.data.alerts.callout_info}} +By default, CockroachDB stores timeseries metrics for the last 30 days, but you can reduce the interval for timeseries storage. Alternatively, if you are exclusively using a third-party tool such as [Prometheus](monitor-cockroachdb-with-prometheus.html) for timeseries monitoring, you can disable timeseries storage entirely. For more details, see this [FAQ](operational-faqs.html#can-i-reduce-or-disable-the-storage-of-timeseries-data). +{{site.data.alerts.end}} + +#### Change time range + +You can change the time range by clicking on the time window. +CockroachDB Admin UI + +{{site.data.alerts.callout_info}} +The Admin UI shows time in UTC, even if you set a different time zone for your cluster. +{{site.data.alerts.end}} + +#### View metrics for a single node + +By default, the Admin UI displays the metrics for the entire cluster. To view the metrics for an individual node, select the node from the **Graph** dropdown. + +CockroachDB Admin UI + +### Summary panel + +A **Summary** panel of key metrics is displayed to the right of the timeseries graphs. + +CockroachDB Admin UI Summary Panel + +The **Summary** panel provides the following metrics: + +Metric | Description +--------|---- +Total Nodes | The total number of nodes in the cluster. Decommissioned nodes are not included in the Total Nodes count.

You can further drill down into the nodes details by clicking on [**View nodes list**](admin-ui-cluster-overview-page.html#node-list). +Capacity Used | The storage capacity used as a percentage of [usable capacity](admin-ui-cluster-overview-page.html#capacity-metrics) allocated across all nodes. +Unavailable Ranges | The number of unavailable ranges in the cluster. A non-zero number indicates an unstable cluster. +Queries per second | The total number of `SELECT`, `UPDATE`, `INSERT`, and `DELETE` queries executed per second across the cluster. +P50 Latency | The 50th percentile of service latency. Service latency is calculated as the time between when the cluster receives a query and finishes executing the query. This time does not include returning results to the client. +P99 Latency | The 99th percentile of service latency. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} +{{site.data.alerts.end}} + +### Events panel + +Underneath the [Summary panel](#summary-panel), the **Events** panel lists the 5 most recent events logged for all nodes across the cluster. To list all events, click **View all events**. + +CockroachDB Admin UI Events + +The following types of events are listed: + +- Database created +- Database dropped +- Table created +- Table dropped +- Table altered +- Index created +- Index dropped +- View created +- View dropped +- Schema change reversed +- Schema change finished +- Node joined +- Node decommissioned +- Node restarted +- Cluster setting changed + +### Statements page + +Click **Statements** on the left to open the [Statements page](admin-ui-statements-page.html). This page helps you identify frequently executed or high latency [SQL statements](sql-statements.html), view SQL statement details, and download SQL statement diagnostics for troubleshooting. + +### Network Latency page + +Click **Network Latency** on the left to open the [Network Latency page](admin-ui-network-latency-page.html). This page displays round-trip latencies between all nodes in your cluster. Latency is the time required to transmit a packet across a network, and is highly dependent on your network topology. Use this page to determine whether your latency is appropriate for your [topology pattern](topology-patterns.html), or to identify nodes with unexpected latencies. + +CockroachDB Admin UI Network Latency matrix + +## See also + +- [Admin UI Overview](admin-ui-overview.html) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) \ No newline at end of file diff --git a/v20.2/admin-ui-cdc-dashboard.md b/v20.2/admin-ui-cdc-dashboard.md new file mode 100644 index 00000000000..67a98050d05 --- /dev/null +++ b/v20.2/admin-ui-cdc-dashboard.md @@ -0,0 +1,74 @@ +--- +title: Changefeeds Dashboard +summary: The Changefeeds dashboard lets you monitor the changefeeds created across your cluster. +toc: true +--- + +The **Changefeeds** dashboard in the CockroachDB Admin UI lets you monitor the [changefeeds](change-data-capture.html) created across your cluster. + +To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** in the left-hand navigation, and select **Dashboard** > **Changefeeds**. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Changefeeds** dashboard displays the following time series graphs: + +## Max Changefeed Latency + +This graph shows the maximum latency for resolved timestamps of any running changefeed. + +CockroachDB Admin UI Max Changefeed Latency graph + +{{site.data.alerts.callout_info}} +The maximum latency for resolved timestamps is distinct from and slower than the commit-to-emit latency for individual change messages. For more information about resolved timestamps, see [Ordering guarantees](change-data-capture.html#ordering-guarantees). +{{site.data.alerts.end}} + +## Sink Byte Traffic + +This graph shows the number of bytes emitted by CockroachDB into the sink for changefeeds. + +CockroachDB Admin UI Sink Byte Traffic graph + +Metric | Description +--------|---- +**Emitted Bytes** | The number of bytes emitted by CockroachDB into the sink for changefeeds. + +## Sink Counts + +This graph shows: + +- The number of messages that CockroachDB sent to the sink. +- The number of flushes that the sink performed for changefeeds. + +CockroachDB Admin UI Sink Counts graph + +Metric | Description +--------|---- +**Messages** | The number of messages that CockroachDB sent to the sink for changefeeds. +**Flushes** | The the number of flushes that the sink performed for changefeeds. + +## Sink Timings + +This graph shows: + +- The time in milliseconds per second required by CockroachDB to send messages to the sink. +- The time CockroachDB spent waiting for the sink to flush the messages for changefeeds. + +CockroachDB Admin UI Sink Timings graph + +Metric | Description +--------|---- +**Message Emit Time** | The time in milliseconds per second required by CockroachDB to send messages to the sink for changefeeds. +**Flush Time** | The time in milliseconds per second that CockroachDB spent waiting for the sink to flush the messages for changefeeds. + +## Changefeed Restarts + +This graph displays the number of times changefeeds restarted due to retryable errors. + +CockroachDB Admin UI Changefeed Restarts graph + +## See also + +- [Change Data Capture](change-data-capture.html) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-cluster-overview-page.md b/v20.2/admin-ui-cluster-overview-page.md new file mode 100644 index 00000000000..d9181034a17 --- /dev/null +++ b/v20.2/admin-ui-cluster-overview-page.md @@ -0,0 +1,146 @@ +--- +title: Cluster Overview Page +toc: true +--- + +The **Cluster Overview** page of the Admin UI displays key metrics about your cluster and individual nodes. These include: + +- Liveness status +- Replication status +- Uptime +- Hardware usage + +If you have an [enterprise license](enterprise-licensing.html), you can enable the [Node Map](#node-map-enterprise) view for a visual representation of your cluster's geographic layout. + +{{site.data.alerts.callout_success}} +Enter your email in the banner at the top to receive updates on CockroachDB [releases](../releases/). +{{site.data.alerts.end}} + +## Cluster Overview panel + +Use the **Cluster Overview** panel to quickly assess the capacity and health of your cluster. + +CockroachDB Admin UI cluster overview + +Metric | Description +--------|---- +Capacity Usage |
  • Used: The total disk space in use by CockroachDB data across all nodes. This excludes the disk space used by the Cockroach binary, operating system, and other system files.
  • Usable: The total disk space usable by CockroachDB data across all nodes. This cannot exceed the store size, if one has been set using [`--store`](cockroach-start.html#store).
See [Capacity metrics](#capacity-metrics) for details on how these values are calculated. +Node Status |
  • The number of `LIVE` nodes in the cluster.
  • The number of `SUSPECT` nodes in the cluster. A node is considered suspect if its [liveness status is unavailable](cluster-setup-troubleshooting.html#node-liveness-issues) or the node is in the process of [decommissioning](#decommissioned-nodes).
  • The number of `DEAD` nodes in the cluster.
  • +Replication Status |
    • The total number of [ranges](architecture/overview.html#glossary) in the cluster.
    • The number of [under-replicated ranges](admin-ui-replication-dashboard.html#review-of-cockroachdb-terminology) in the cluster. A non-zero number indicates an unstable cluster.
    • The number of [unavailable ranges](admin-ui-replication-dashboard.html#review-of-cockroachdb-terminology) in the cluster. A non-zero number indicates an unstable cluster.
    • + +### Capacity metrics + +The [Cluster Overview](#cluster-overview-panel), [Node List](#node-list), and [Node Map](#node-map-enterprise) display **Capacity Usage** by the CockroachDB [store](architecture/storage-layer.html) (the directory on each node where CockroachDB reads and writes its data) as a percentage of the disk space that is **usable** on the cluster, locality, or node. + +Usable disk space is constrained by the following: + +- The maximum store size, which may be specified using the [`--store`](cockroach-start.html#store) flag when starting nodes. If no store size has been explicitly set, the actual disk capacity is used as the limit. This value is displayed on the Capacity graph in the [Storage dashboard](admin-ui-storage-dashboard.html#capacity). +- Any disk space occupied by non-CockroachDB data. This may include the operating system and other system files, as well as the Cockroach binary itself. + +The Admin UI thus calculates **usable** disk space as the sum of empty disk space, up to the value of the maximum store size, and disk space that is already being **used** by CockroachDB data. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} +{{site.data.alerts.end}} + +## Node List + +The **Node List** groups nodes by locality. The lowest-level locality tier is used to organize the Node List. Hover over a locality to see all localities for the group of nodes. + +{{site.data.alerts.callout_success}} +We recommend [defining `--locality` flags when starting nodes](cockroach-start.html#locality). CockroachDB uses locality to distribute replicas and mitigate [network latency](admin-ui-network-latency-page.html). Locality is also a prerequisite for enabling the [Node Map](#node-map-enterprise). +{{site.data.alerts.end}} + +### Node status + +Each locality and node is displayed with its current operational status. + +Locality Status | Description +-------|------------ +`LIVE` | All nodes in the locality are live. +`WARNING` | Locality has 1 or more `SUSPECT`, `DECOMMISSIONING`, or `DEAD` nodes (red indicates a dead node). + +Node Status | Description +-------|------------ +`LIVE` | Node is online and updating its liveness record. +`SUSPECT` | Node has an [unavailable liveness status](cluster-setup-troubleshooting.html#node-liveness-issues). +`DECOMMISSIONING` | Node is in the [process of decommissioning](remove-nodes.html#how-it-works). +`DECOMMISSIONED` | Node has been decommissioned for permanent removal from the cluster. +`DEAD` | Node has not [updated its liveness record](cluster-setup-troubleshooting.html#node-liveness-issues) for 5 minutes. + +{{site.data.alerts.callout_info}} +Nodes are considered dead once they have not [updated their liveness record](cluster-setup-troubleshooting.html#node-liveness-issues) for a certain amount of time (5 minutes by default). At this point, the [automated repair process](cockroach-quit.html#how-it-works) starts, wherein CockroachDB rebalances replicas from dead nodes to live nodes, using the unaffected replicas as sources. +{{site.data.alerts.end}} + +### Node details + +The following details are also shown. + +CockroachDB Admin UI node list + +Column | Description +-------|------------ +Node Count | Number of nodes in the locality. +Nodes | Nodes are grouped by locality and displayed with their address. Click the address to view node statistics. Hover over a row and click **Logs** to see the node's log. +Uptime | Amount of time the node has been running. +Replicas | Number of replicas on the node or in the locality. +Capacity Usage | Percentage of usable disk space occupied by CockroachDB data on the node or in the locality. See [Capacity metrics](#capacity-metrics). +Memory Usage | Memory used by CockroachDB as a percentage of the total memory on the node or in the locality. +CPUs | Number of vCPUs on the machine. +Version | Build tag of the CockroachDB version installed on the node. + +### Decommissioned Nodes + +Nodes that have recently been decommissioned for permanent removal from the cluster are listed in the table of **Recently Decommissioned Nodes**. You can see the full history of decommissioned nodes by clicking "View all decommissioned nodes". + +CockroachDB Admin UI node list + +{{site.data.alerts.callout_info}} +When you [decommission a node](remove-nodes.html), CockroachDB lets the node finish in-flight requests, rejects any new requests, and transfers all range replicas and range leases off the node so that it can be safely shut down. +{{site.data.alerts.end}} + +## Node Map (Enterprise) + +The **Node Map** is an [enterprise](enterprise-licensing.html) feature that visualizes the geographical configuration of your cluster. It requires that [`--locality` flags have been defined](cockroach-start.html#locality) for your nodes. + +For guidance on enabling and configuring the node map, see [Enable the Node Map](enable-node-map.html). + +CockroachDB Admin UI Summary Panel + +The Node Map uses the longitude and latitude of each locality to position the components on the map. The map is populated with [**locality components**](#locality-component) and [**node components**](#node-component). + +### Locality component + +A locality component represents capacity, CPU, and QPS metrics for a given locality. + +The map shows the components for the highest-level locality tier (e.g., region). You can click on the **Node Count** of a locality component to view any lower-level localities (e.g., availability zone). + +For details on how **Capacity Usage** is calculated, see [Capacity metrics](#capacity-metrics). + +CockroachDB Admin UI Summary Panel + +{{site.data.alerts.callout_info}} +On multi-core systems, the displayed CPU usage can be greater than 100%. Full utilization of 1 core is considered as 100% CPU usage. If you have _n_ cores, then CPU usage can range from 0% (indicating an idle system) to (_n_ * 100)% (indicating full utilization). +{{site.data.alerts.end}} + +### Node component + +A node component represents capacity, CPU, and QPS metrics for a given node. + +Node components are accessed by clicking on the **Node Count** of the lowest-level [locality component](#locality-component). + +For details on how **Capacity Usage** is calculated, see [Capacity metrics](#capacity-metrics). + +CockroachDB Admin UI Summary Panel + +{{site.data.alerts.callout_info}} +On multi-core systems, the displayed CPU usage can be greater than 100%. Full utilization of 1 core is considered as 100% CPU usage. If you have _n_ cores, then CPU usage can range from 0% (indicating an idle system) to (_n_ * 100)% (indicating full utilization). +{{site.data.alerts.end}} + +## See also + +- [Production Checklist](recommended-production-settings.html) +- [Locality](cockroach-start.html#locality) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-custom-chart-debug-page.md b/v20.2/admin-ui-custom-chart-debug-page.md new file mode 100644 index 00000000000..47a61ad59c9 --- /dev/null +++ b/v20.2/admin-ui-custom-chart-debug-page.md @@ -0,0 +1,59 @@ +--- +title: Custom Chart Debug Page +toc: true +--- + +The **Custom Chart** debug page in the Admin UI can be used to create one or multiple custom charts showing any combination of over [200 available metrics](#available-metrics). + +The definition of the customized dashboard is encoded in the URL. To share the dashboard with someone, send them the URL. Like any other URL, it can be bookmarked, sit in a pinned tab in your browser, etc. + + +## Accessing the **Custom Chart** page + +To access the **Custom Chart** debug page, [access the Admin UI](admin-ui-access-and-navigate.html), and either: + +- Open http://localhost:8080/#/debug/chart in your browser (replacing `localhost` and `8080` with your node's host and port). + +- Click the gear icon on the left to access the **Advanced Debugging Page**. In the **Reports** section, click **Custom TimeSeries Chart**. + +## Using the **Custom Chart** page + +CockroachDB Admin UI + +On the **Custom Chart** page, you can set the time span for all charts, add new custom charts, and customize each chart: + +- To set the time span for the page, use the dropdown menu above the charts and select the desired time span. + +- To add a chart, click **Add Chart** and customize the new chart. + +- To customize each chart, use the **Units** dropdown menu to set the units to display. Then use the table below the chart to select the metrics being queried, and how they'll be combined and displayed. Options include: +{% include {{page.version.version}}/admin-ui-custom-chart-debug-page-00.html %} + +## Examples + +### Query user and system CPU usage + +CockroachDB Admin UI + +To compare system vs. userspace CPU usage, select the following values under **Metric Name**: + +- `sys.cpu.sys.percent` +- `sys.cpu.user.percent` + +The Y-axis label is the **Count**. A count of 1 represents 100% utilization. The **Aggregator** of **Sum** can show the count to be above 1, which would mean CPU utilization is greater than 100%. + +Checking **Per Node** displays statistics for each node, which could show whether an individual node's CPU usage was higher or lower than the average. + +## Available metrics + +{{site.data.alerts.callout_info}} +This list is taken directly from the source code and is subject to change. Some of the metrics listed below are already visible in other areas of the [Admin UI](admin-ui-overview.html). +{{site.data.alerts.end}} + +{% include {{page.version.version}}/metric-names.md %} + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-databases-page.md b/v20.2/admin-ui-databases-page.md new file mode 100644 index 00000000000..21a38927420 --- /dev/null +++ b/v20.2/admin-ui-databases-page.md @@ -0,0 +1,61 @@ +--- +title: Database Page +toc: true +--- + +{{site.data.alerts.callout_info}} +On a secure cluster, this area of the Admin UI can only be accessed by an `admin` user. See [Admin UI access](admin-ui-overview.html#admin-ui-access). +{{site.data.alerts.end}} + +The **Databases** page of the Admin UI provides details of the following: + +- The databases configured. +- The tables in each database. +- The grants assigned to each user. + +To view this page, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Databases** in the left-hand navigation. + +## Tables view + +The **Tables** view shows details of the system table as well as the tables in your databases. + +To view [table details](#table-details), click on a table name. + +CockroachDB Admin UI Database Tables View + +The following are displayed for each table: + +Parameter | Description +--------|---- +Table Name | The name of the table. +Size | Approximate disk size of all replicas of this table on the cluster. +Ranges | The number of ranges in the table. +\# of Columns | The number of columns in the table. +\# of Indices | The number of indices for the table. + +### Table details + +Click any table name in [Tables](#tables-view) view to display details for that table. + +CockroachDB Admin UI Database Tables View + +- **Overview** displays the SQL statements used to [create and define the table](create-table.html), as well as [partitioning](partitioning.html) info and [zone configurations](configure-replication-zones.html). In addition, the following metrics are displayed: + - **Size** is the approximate disk size of all replicas of this table on the cluster. + - **Ranges** is the number of [ranges](architecture/overview.html#terms) in this table. + - **Replicas** is the number of [replicas](architecture/replication-layer.html) of this table on the cluster. +- **Grants** displays the [grants](#grants-view) associated with the table. + +## Grants view + +The **Grants** view shows the [privileges](authorization.html#assign-privileges) granted to users for each database. + +For more details about grants and privileges, see [`GRANT `](grant.html). + +CockroachDB Admin UI Database Grants View + +## See also + +- [Statements page](admin-ui-statements-page.html) +- [Assign privileges](authorization.html#assign-privileges) +- [`GRANT `](grant.html) +- [Raw status endpoints](monitoring-and-alerting.html#raw-status-endpoints) \ No newline at end of file diff --git a/v20.2/admin-ui-debug-pages.md b/v20.2/admin-ui-debug-pages.md new file mode 100644 index 00000000000..74022886260 --- /dev/null +++ b/v20.2/admin-ui-debug-pages.md @@ -0,0 +1,53 @@ +--- +title: Advanced Debug Page +toc: true +--- + +The **Advanced Debug** page of the Admin UI provides links to advanced monitoring and troubleshooting reports and cluster configuration details. To view this page, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Advanced Debug** in the left-hand navigation. + +{{site.data.alerts.callout_info}} +These pages are experimental and undocumented. If you find an issue, let us know through [these channels](https://www.cockroachlabs.com/community/). + {{site.data.alerts.end}} + +## License and node information + +On the right-side of the page, the following information is displayed: + +- CockroachDB license type: Helps determine if you have access to Enterprise features. +- Current node ID: Helps identify the current node when viewing the Admin UI through a load balancer. + +## Reports and Configuration + +The following debug reports and configuration views are useful for monitoring and troubleshooting CockroachDB: + +Report | Description | Access level +--------|-----|-------- +[Custom Time Series Chart](admin-ui-custom-chart-debug-page.html) | Create a custom chart of time series data. | All users +Problem Ranges | View ranges in your cluster that are unavailable, underreplicated, slow, or have other problems. | [`admin` users only on secure clusters](admin-ui-overview.html#admin-ui-access) +Network Latency | Check latencies between all nodes in your cluster. | All users +Data Distribution and Zone Configs | View the distribution of table data across nodes and verify zone configuration. | [`admin` users only on secure clusters](admin-ui-overview.html#admin-ui-access) +Cluster Settings | View cluster settings and their configured values. | All users can view data according to their privileges +Localities | Check node localities for your cluster. | [`admin` users only on secure clusters](admin-ui-overview.html#admin-ui-access) + +## Even More Advanced Debugging + +The **Even More Advanced Debugging** section of the page lists additional reports that are largely internal and intended for use by CockroachDB developers. You can ignore this section while monitoring and troubleshooting CockroachDB. Alternatively, if you want to learn how to use these pages, feel free to contact us through [these channels](https://www.cockroachlabs.com/community/). + +## Raw Status Endpoints (JSON) + +Depending on your [access level](admin-ui-overview.html#admin-ui-access), the endpoints listed here provide access to: + +- [Log files](debug-and-error-logs.html#write-to-file) +- Secondary log files (e.g., RocksDB logs, [execution logs](query-behavior-troubleshooting.html#cluster-wide-execution-logs), [slow query logs](query-behavior-troubleshooting.html#using-the-slow-query-log), [authentication logs](query-behavior-troubleshooting.html#authentication-logs)) +- Node status +- Hot ranges +- Node-specific metrics +- Session data +- Cluster-wide range data +- Allocator runs + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-hardware-dashboard.md b/v20.2/admin-ui-hardware-dashboard.md new file mode 100644 index 00000000000..b733ff3de36 --- /dev/null +++ b/v20.2/admin-ui-hardware-dashboard.md @@ -0,0 +1,128 @@ +--- +title: Hardware Dashboard +summary: The Hardware dashboard lets you monitor CPU usage, disk throughput, network traffic, storage capacity, and memory. +toc: true +--- + +The **Hardware** dashboard lets you monitor the hardware utilization of your cluster. This includes CPU usage, disk throughput, network traffic, storage capacity, and memory. + +To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** in the left-hand navigation, and select **Dashboard** > **Hardware**. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Hardware** dashboard displays the following time series graphs: + +## CPU Percent + +CockroachDB Admin UI CPU Percent graph + +{{site.data.alerts.callout_info}} +This graph shows the CPU consumption by the CockroachDB process only and is useful as long as there are no other processes consuming significant CPU on the node. In case you have other processes running on the node, use a separate monitoring tool to measure the total CPU consumption across all processes. +{{site.data.alerts.end}} + +- In the node view, the graph shows the percentage of CPU in use by the CockroachDB process for the selected node. + +- In the cluster view, the graph shows the percentage of CPU in use by the CockroachDB process across all nodes. + +{{site.data.alerts.callout_info}} +For multi-core systems, the percentage of CPU usage is calculated by normalizing the CPU usage across all cores, whereby 100% utilization indicates that all cores are fully utilized. +{{site.data.alerts.end}} + +## Memory Usage + +CockroachDB Admin UI Memory Usage graph + +{{site.data.alerts.callout_info}} +This graph shows the memory consumption by the CockroachDB process only and is useful as long as there are no other processes consuming significant memory on the node. In case you have other processes running on the node, use a separate monitoring tool to measure the total memory consumption across all processes. +{{site.data.alerts.end}} + +- In the node view, the graph shows the memory in use by CockroachDB for the selected node. + +- In the cluster view, the graph shows the memory in use by CockroachDB across all nodes in the cluster. + +## Disk Read Bytes + +CockroachDB Admin UI Disk Read Bytes graph + +- In the node view, the graph shows the 10-second average of the number of bytes read per second by all processes, including CockroachDB, for the selected node. + +- In the cluster view, the graph shows the 10-second average of the number of bytes read per second by all processes, including CockroachDB, across all nodes. + +## Disk Write Bytes + +CockroachDB Admin UI Disk Write Bytes graph + +- In the node view, the graph shows the 10-second average of the number of bytes written per second by all processes, including CockroachDB, for the node. + +- In the cluster view, the graph shows the 10-second average of the number of bytes written per second by all processes, including CockroachDB, across all nodes. + +## Disk Read Ops + +CockroachDB Admin UI Disk Read Ops graph + +- In the node view, the graph shows the 10-second average of the number of disk read ops per second for all processes, including CockroachDB, for the selected node. + +- In the cluster view, the graph shows the 10-second average of the number of disk read ops per second for all processes, including CockroachDB, across all nodes. + +## Disk Write Ops + +CockroachDB Admin UI Disk Write Ops graph + +- In the node view, the graph shows the 10-second average of the number of disk write ops per second for all processes, including CockroachDB, for the node. + +- In the cluster view, the graph shows the 10-second average of the number of disk write ops per second for all processes, including CockroachDB, across all nodes. + +## Disk IOPS in Progress + +CockroachDB Admin UI Disk IOPS in Progress graph + +- In the node view, the graph shows the number of disk reads and writes in queue for all processes, including CockroachDB, for the selected node. + +- In the cluster view, the graph shows the number of disk reads and writes in queue for all processes, including CockroachDB, across all nodes in the cluster. + +{{site.data.alerts.callout_info}} +For Mac OS, this graph is not populated and shows zero disk IOPS in progress. This is a [known limitation](https://github.com/cockroachdb/cockroach/issues/27927) that may be lifted in the future. +{{site.data.alerts.end}} + +## Available Disk Capacity + +CockroachDB Admin UI Disk Capacity graph + +Metric | Description +--------|-------- +**Available Disk Capacity** | Free disk space available to CockroachDB data on each node. + +### Capacity metrics + +The **available** disk capacity equals the amount of empty disk space, up to the value of the maximum store size. The store size is determined as follows: + +- If a store size was specified using the [`--store`](cockroach-start.html#store) flag when starting nodes, this value is used as the limit for CockroachDB data. +- If no store size has been explicitly set, the actual disk capacity is used as the limit for CockroachDB data. + +The disk usage of the Cockroach binary, operating system, and other system files is not shown on the **Available Disk Capacity** graph. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} +{{site.data.alerts.end}} + +## Network Bytes Received + +CockroachDB Admin UI Network Bytes Received graph + +- In the node view, the graph shows the 10-second average of the number of network bytes received per second for all processes, including CockroachDB, for the node. + +- In the cluster view, the graph shows the 10-second average of the number of network bytes received for all processes, including CockroachDB, per second across all nodes. + +## Network Bytes Sent + +CockroachDB Admin UI Network Bytes Sent graph + +- In the node view, the graph shows the 10-second average of the number of network bytes sent per second by all processes, including CockroachDB, for the node. + +- In the cluster view, the graph shows the 10-second average of the number of network bytes sent per second by all processes, including CockroachDB, across all nodes. + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-jobs-page.md b/v20.2/admin-ui-jobs-page.md new file mode 100644 index 00000000000..9215fb79551 --- /dev/null +++ b/v20.2/admin-ui-jobs-page.md @@ -0,0 +1,80 @@ +--- +title: Jobs Page +toc: true +--- + +The **Jobs** page of the Admin UI provides details about long-running tasks performed by your cluster. These can include: + +{% include {{ page.version.version }}/sql/schema-changes.md %}. +- [`IMPORT`](import.html). +- Enterprise [`BACKUP`](backup.html) and [`RESTORE`](restore.html). +- [User-created table statistics](create-statistics.html) created for use by the [cost-based optimizer](cost-based-optimizer.html). +- [Automatic table statistics](cost-based-optimizer.html#table-statistics). +- [Changefeeds](change-data-capture.html). + +{{site.data.alerts.callout_success}} +All users can see their own jobs, and `admin` users can view all jobs performed across all nodes in the cluster. +{{site.data.alerts.end}} + +To view these details, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Jobs** in the left-hand navigation. + +## Filter jobs + +Use the **Status** menu to filter jobs by [job status](#job-status). + +Use the **Type** menu to filter jobs by type. + +You can toggle between showing the latest 50 jobs or all jobs on the cluster. + +{{site.data.alerts.callout_info}} +Jobs are deleted every 14 days. This interval can be changed via the `jobs.retention_time` [cluster setting](cluster-settings.html). + +The Jobs list is designed for you to manage pending work. It is not intended to display the canonical record of all jobs that have run. If you need a historical record of all jobs you have run, you should log this information externally. +{{site.data.alerts.end}} + +## Jobs list + +Use the **Jobs** list to see your recently created and completed jobs. + +- For changefeeds, the table displays a [high-water timestamp that advances as the changefeed progresses](change-data-capture.html#monitor-a-changefeed). This is a guarantee that all changes before or at the timestamp have been emitted. Hover over the high-water timestamp to view the [system time](as-of-system-time.html). + +- [Automatic table statistics](cost-based-optimizer.html#table-statistics) jobs are not displayed even when the **Type** menu is set to **All**. To view these jobs, set **Type** to **Automatic-Statistics Creation** as described [above](#filter-jobs). + +- To view [job details](#job-details), click on the job description. + +CockroachDB Admin UI Jobs Page + +Parameter | Description +----------|------------ +Description | SQL statement that created the job. +Job ID | Unique job ID. This value is used to [pause](pause-job.html), [resume](resume-job.html), or [cancel](cancel-job.html) jobs. +Users | User that created the job. +Creation Time | Date and time the job was created. +Status | Current [job status](#job-status) or completion progress. + +### Job status + +Status | Description +-------|------------ +`PENDING` | Job is created but has not started running. +`PAUSED` | Job is [paused](pause-job.html). +`FAILED` | Job failed to complete. +`SUCCEEDED` | Job successfully completed. +`CANCELED` | Job was [cancelled](cancel-job.html). + +A job that is currently running will be displayed with its percent completion and time remaining, rather than the `RUNNING` status. + +## Job details + +Click any description on the [jobs list](#jobs-list) to see the full SQL statement that created the job. + +The job ID, creation time, users, and status are also shown. + +CockroachDB Admin UI Jobs Page + +## See also + +- [`SHOW JOBS`](show-jobs.html) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) \ No newline at end of file diff --git a/v20.2/admin-ui-network-latency-page.md b/v20.2/admin-ui-network-latency-page.md new file mode 100644 index 00000000000..728949d994a --- /dev/null +++ b/v20.2/admin-ui-network-latency-page.md @@ -0,0 +1,57 @@ +--- +title: Network Latency Page +toc: true +--- + +The **Network Latency** page displays round-trip latencies between all nodes in your cluster. Latency is the time required to transmit a packet across a network, and is highly dependent on your network topology. Use this page to determine whether your latency is appropriate for your [topology pattern](topology-patterns.html), or to identify nodes with unexpected latencies. + +To view this page, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Network Latency** in the left-hand navigation. + +## Sort and filter network latency + +Use the **Sort By** menu to arrange the latency matrix by [locality](cockroach-start.html#locality) (e.g., cloud, region, availability zone, datacenter). + +Use the **Filter** menu to select specific nodes or localities to view. + +Select **Collapse Nodes** to display the mean latencies of each locality, depending on how the matrix is sorted. This is a way to quickly assess cross-regional or cross-cloud latency. + +## Understanding the Network Latency matrix + +Each cell in the matrix displays the round-trip latency in milliseconds between two nodes in your cluster. Round-trip latency includes the return time of a packet. Latencies are color-coded by their standard deviation from the mean latency on the network: green for lower values, and blue for higher. + +CockroachDB Admin UI Network Latency matrix + +Rows represent origin nodes, and columns represent destination nodes. Hover over a cell to see round-trip latency and locality metadata for origin and destination nodes. + +On a [typical multi-region cluster](demo-low-latency-multi-region-deployment.html#step-4-access-the-admin-ui), you can expect much lower latencies between nodes in the same region/availability zone. Nodes in different regions/availability zones, meanwhile, will experience higher latencies that reflect their geographical distribution. + +For instance, the cluster shown above has nodes in `us-west1`, `us-east1`, and `europe-west2`. Latencies are highest between nodes in `us-west1` and `europe-west2`, which span the greatest distance. This is especially clear when sorting by region or availability zone and collapsing nodes: + +CockroachDB Admin UI Network Latency collapsed nodes + +### No connections + +Nodes that have lost a connection are displayed in a separate color. This can help you locate a network partition in your cluster. + +{{site.data.alerts.callout_info}} +A network partition prevents nodes from communicating with each other in one or both directions. This can be due to a configuration problem with the network, such as when whitelisted IP addresses or hostnames change after a node is torn down and rebuilt. In a symmetric partition, node communication is broken in both directions. In an asymmetric partition, node communication works in one direction but not the other. + +The effect of a network partition depends on which nodes are partitioned, where the ranges are located, and to a large extent, whether [localities](cockroach-start.html#locality) are defined. If localities are not defined, a partition that cuts off at least (n-1)/2 nodes will cause data unavailability. +{{site.data.alerts.end}} + +Click the **NO CONNECTIONS** link to see lost connections between nodes or [localities](cockroach-start.html#locality), if any are defined. + +## Topology fundamentals + +{% include {{ page.version.version }}/topology-patterns/fundamentals.md %} + +{{site.data.alerts.callout_info}} +Network latency limits the performance of individual operations. You can use the [Statements](admin-ui-statements-page.html) page to see the latencies of SQL statements on gateway nodes. +{{site.data.alerts.end}} + +## See also + +- [Topology Patterns](topology-patterns.html) +- [CockroachDB Performance](performance.html#latency) +- [Performance Tuning](performance-tuning.html) +- [Low Latency Reads and Writes in a Multi-Region Cluster](demo-low-latency-multi-region-deployment.html) \ No newline at end of file diff --git a/v20.2/admin-ui-overview-dashboard.md b/v20.2/admin-ui-overview-dashboard.md new file mode 100644 index 00000000000..c9f0bfdc783 --- /dev/null +++ b/v20.2/admin-ui-overview-dashboard.md @@ -0,0 +1,78 @@ +--- +title: Overview Dashboard +summary: The Overview dashboard lets you monitor important SQL performance, replication, and storage metrics. +toc: true +--- + +The **Overview** dashboard lets you monitor important SQL performance, replication, and storage metrics. To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Metrics** on the left-hand navigation bar. The **Overview** dashboard is displayed by default. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Overview** dashboard displays the following time series graphs: + +## SQL Queries + +CockroachDB Admin UI SQL Queries graph + +- In the node view, the graph shows the 10-second average of the number of `SELECT`/`INSERT`/`UPDATE`/`DELETE` queries per second issued by SQL clients on the node. + +- In the cluster view, the graph shows the sum of the per-node averages, that is, an aggregate estimation of the current query load over the cluster, assuming the last 10 seconds of activity per node are representative of this load. + +## Service Latency: SQL, 99th percentile + +CockroachDB Admin UI Service Latency graph + +Service latency is calculated as the time between when the cluster receives a query and finishes executing the query. This time does not include returning results to the client. + +- In the node view, the graph shows the 99th [percentile](https://en.wikipedia.org/wiki/Percentile#The_normal_distribution_and_percentiles) of service latency for the node. + +- In the cluster view, the graph shows the 99th [percentile](https://en.wikipedia.org/wiki/Percentile#The_normal_distribution_and_percentiles) of service latency across all nodes in the cluster. + +## Replicas per Node + +CockroachDB Admin UI Replicas per node graph + +Ranges are subsets of your data, which are replicated to ensure survivability. Ranges are replicated to a configurable number of CockroachDB nodes. + +- In the node view, the graph shows the number of range replicas on the selected node. + +- In the cluster view, the graph shows the number of range replicas on each node in the cluster. + +For details about how to control the number and location of replicas, see [Configure Replication Zones](configure-replication-zones.html). + +{{site.data.alerts.callout_info}} +The timeseries data used to power the graphs in the Admin UI is stored within the cluster and accumulates for 30 days before it starts getting truncated. As a result, for the first 30 days or so of a cluster's life, you will see a steady increase in disk usage and the number of ranges even if you aren't writing data to the cluster yourself. For more details, see this [FAQ](operational-faqs.html#why-is-disk-usage-increasing-despite-lack-of-writes). +{{site.data.alerts.end}} + +## Capacity + +CockroachDB Admin UI Capacity graph + +You can monitor the **Capacity** graph to determine when additional storage is needed (e.g., by [scaling your cluster](cockroach-start.html)). + +Metric | Description +--------|-------- +**Capacity** | The maximum store size. This value may be set per node using [`--store`](cockroach-start.html#store). If a store size has not been set, this metric displays the actual disk capacity. See [Capacity metrics](#capacity-metrics). +**Available** | The free disk space available to CockroachDB data. +**Used** | The disk space in use by CockroachDB data. This excludes the Cockroach binary, operating system, and other system files. + +### Capacity metrics + +The **Capacity** graph displays disk usage by CockroachDB data in relation to the maximum [store](architecture/storage-layer.html) size, which is determined as follows: + +- If a store size was specified using the [`--store`](cockroach-start.html#store) flag when starting nodes, this value is used as the limit for CockroachDB data. +- If no store size has been explicitly set, the actual disk capacity is used as the limit for CockroachDB data. + +The **available** capacity thus equals the amount of empty disk space, up to the value of the maximum store size. The **used** capacity refers only to disk space occupied by CockroachDB data, which resides in the store directory on each node. + +The disk usage of the Cockroach binary, operating system, and other system files is not shown on the **Capacity** graph. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} +{{site.data.alerts.end}} + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-overview.md b/v20.2/admin-ui-overview.md new file mode 100644 index 00000000000..5edf538f180 --- /dev/null +++ b/v20.2/admin-ui-overview.md @@ -0,0 +1,61 @@ +--- +title: Admin UI Overview +summary: Use the Admin UI to monitor and optimize cluster performance. +toc: true +redirect_from: explore-the-admin-ui.html +key: explore-the-admin-ui.html +--- + +The CockroachDB Admin UI provides details about your cluster and database configuration, and helps you optimize cluster performance. + +## Admin UI areas + +Area | Description +--------|---- +[Node Map](enable-node-map.html) | View and monitor the metrics and geographical configuration of your cluster. +[Cluster Health](admin-ui-access-and-navigate.html#summary-panel) | View essential metrics about the cluster's health, such as the number of live, dead, and suspect nodes, the number of unavailable ranges, and the queries per second and service latency across the cluster. +[Overview Metrics](admin-ui-overview-dashboard.html) | View important SQL performance, replication, and storage metrics. +[Hardware Metrics](admin-ui-hardware-dashboard.html) | View metrics about CPU usage, disk throughput, network traffic, storage capacity, and memory. +[Runtime Metrics](admin-ui-runtime-dashboard.html) | View metrics about node count, CPU time, and memory usage. +[SQL Performance](admin-ui-sql-dashboard.html) | View metrics about SQL connections, byte traffic, queries, transactions, and service latency. +[Storage Utilization](admin-ui-storage-dashboard.html) | View metrics about storage capacity and file descriptors. +[Replication Details](admin-ui-replication-dashboard.html) | View metrics about how data is replicated across the cluster, such as range status, replicas per store, and replica quiescence. +[Changefeed Details](admin-ui-cdc-dashboard.html) | View metrics about the [changefeeds](change-data-capture.html) created across your cluster. +[Nodes Details](admin-ui-access-and-navigate.html#summary-panel) | View details of live, dead, and decommissioned nodes. +[Events](admin-ui-access-and-navigate.html#events-panel) | View a list of recent cluster events. +[Database Details](admin-ui-databases-page.html) | View details about the system and user databases in the cluster. +[Statements Details](admin-ui-statements-page.html) | Identify frequently executed or high latency [SQL statements](sql-statements.html) +[Network Latency](admin-ui-network-latency-page.html) | View latencies and lost connections between all nodes in your cluster. +[Jobs Details](admin-ui-jobs-page.html) | View details of jobs running in the cluster. +[Advanced Debugging Pages](admin-ui-debug-pages.html) | View advanced monitoring and troubleshooting reports. These include details about data distribution, the state of specific queues, and slow query metrics. These details are largely intended for use by CockroachDB developers. + +## Admin UI access + +On insecure clusters, all areas of the Admin UI are accessible to all users. + +On secure clusters, certain areas of the Admin UI can only be accessed by [`admin` users](authorization.html#admin-role). These areas display information from privileged HTTP endpoints that operate with `admin` privilege. + +For security reasons, non-admin users access only the data over which they have privileges (e.g., their tables, jobs, and list of sessions), and data that does not require privileges (e.g., cluster health, node status, metrics). + +{{site.data.alerts.callout_info}} +The default `root` user is a member of the `admin` role. Use the following command to [grant users membership to the `admin` role](grant-roles.html): + +GRANT admin TO \; +{{site.data.alerts.end}} + +Secure area | Privileged information +-----|----- +[Node Map](enable-node-map.html) | Database and table names +[Database Details](admin-ui-databases-page.html) | Stored table data +[Statements Details](admin-ui-statements-page.html) | SQL statements +[Advanced Debugging Pages](admin-ui-debug-pages.html) (some reports) | Stored table data, operational details, internal IP addresses, names, credentials, application data (depending on report) + +{{site.data.alerts.callout_info}} +By default, the Admin UI shares anonymous usage details with Cockroach Labs. For information about the details shared and how to opt-out of reporting, see [Diagnostics Reporting](diagnostics-reporting.html). +{{site.data.alerts.end}} + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-replication-dashboard.md b/v20.2/admin-ui-replication-dashboard.md new file mode 100644 index 00000000000..3d729b762e0 --- /dev/null +++ b/v20.2/admin-ui-replication-dashboard.md @@ -0,0 +1,111 @@ +--- +title: Replication Dashboard +summary: The Replication dashboard lets you monitor the replication metrics for your cluster. +toc: true +--- + +The **Replication** dashboard in the CockroachDB Admin UI enables you to monitor the replication metrics for your cluster. + +To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** in the left-hand navigation, and select **Dashboard** > **Replication**. + +## Review of CockroachDB terminology + +- **Range**: CockroachDB stores all user data and almost all system data in a giant sorted map of key-value pairs. This keyspace is divided into "ranges", contiguous chunks of the keyspace, so that every key can always be found in a single range. +- **Range Replica:** CockroachDB replicates each range (3 times by default) and stores each replica on a different node. +- **Range Lease:** For each range, one of the replicas holds the "range lease". This replica, referred to as the "leaseholder", is the one that receives and coordinates all read and write requests for the range. +- **Under-replicated Ranges:** When a cluster is first initialized, the few default starting ranges will only have a single replica, but as soon as other nodes are available, they will replicate to them until they've reached their desired replication factor, the default being 3. If a range does not have enough replicas, the range is said to be "under-replicated". +- **Unavailable Ranges:** If a majority of a range's replicas are on nodes that are unavailable, then the entire range is unavailable and will be unable to process queries. + +For more details, see [Scalable SQL Made Easy: How CockroachDB Automates Operations](https://www.cockroachlabs.com/blog/automated-rebalance-and-repair/) + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Replication** dashboard displays the following time series graphs: + +## Ranges + +CockroachDB Admin UI Replicas per Store + +The **Ranges** graph shows you various details about the status of ranges. + +- In the node view, the graph shows details about ranges on the node. + +- In the cluster view, the graph shows details about ranges across all nodes in the cluster. + +On hovering over the graph, the values for the following metrics are displayed: + +Metric | Description +--------|---- +Ranges | The number of ranges. +Leaders | The number of ranges with leaders. If the number does not match the number of ranges for a long time, troubleshoot your cluster. +Lease Holders | The number of ranges that have leases. +Leaders w/o Leases | The number of Raft leaders without leases. If the number if non-zero for a long time, troubleshoot your cluster. +Unavailable | The number of unavailable ranges. If the number if non-zero for a long time, troubleshoot your cluster. +Under-replicated | The number of under-replicated ranges. + +## Logical Bytes per Store + +CockroachDB Admin UI Replicas per Store + +Metric | Description +--------|-------- +**Logical Bytes per Store** | Number of logical bytes stored in [key-value pairs](architecture/distribution-layer.html#table-data) on each node. This includes historical and deleted data. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/admin-ui/logical-bytes.md %} +{{site.data.alerts.end}} + +## Replicas Per Store + +CockroachDB Admin UI Replicas per Store + +- In the node view, the graph shows the number of range replicas on the store. + +- In the cluster view, the graph shows the number of range replicas on each store. + +You can [Configure replication zones](configure-replication-zones.html) to set the number and location of replicas. You can monitor the configuration changes using the Admin UI, as described in [Fault tolerance and recovery](demo-fault-tolerance-and-recovery.html). + +## Replica Quiescence + +CockroachDB Admin UI Replica Quiescence + +- In the node view, the graph shows the number of replicas on the node. + +- In the cluster view, the graph shows the number of replicas across all nodes. + +On hovering over the graph, the values for the following metrics are displayed: + +Metric | Description +--------|---- +Replicas | The number of replicas. +Quiescent | The number of replicas that haven't been accessed for a while. + +## Snapshots + +CockroachDB Admin UI Replica Snapshots + +Usually the nodes in a [Raft group](architecture/replication-layer.html#raft) stay synchronized by following along the log message by message. However, if a node is far enough behind the log (e.g., if it was offline or is a new node getting up to speed), rather than send all the individual messages that changed the range, the cluster can send it a snapshot of the range and it can start following along from there. Commonly this is done preemptively, when the cluster can predict that a node will need to catch up, but occasionally the Raft protocol itself will request the snapshot. + +Metric | Description +-------|------------ +Generated | The number of snapshots created per second. +Applied (Raft-initiated) | The number of snapshots applied to nodes per second that were initiated within Raft. +Applied (Learner) | The number of snapshots applied to nodes per second that were anticipated ahead of time (e.g., because a node was about to be added to a Raft group). This metric replaces the `Applied (Preemptive)` metric in 19.2 and onwards. +Applied (Preemptive) | The number of snapshots applied to nodes per second that were anticipated ahead of time (e.g., because a node was about to be added to a Raft group). This metric was used in pre-v19.2 releases and will be removed in future releases. +Reserved | The number of slots reserved per second for incoming snapshots that will be sent to a node. + +## Other graphs + +The **Replication** dashboard shows other time series graphs that are important for CockroachDB developers: + +- Leaseholders per Store +- Average Queries per Store +- Range Operations + +For monitoring CockroachDB, it is sufficient to use the [**Ranges**](#ranges), [**Replicas per Store**](#replicas-per-store), and [**Replica Quiescence**](#replica-quiescence) graphs. + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-runtime-dashboard.md b/v20.2/admin-ui-runtime-dashboard.md new file mode 100644 index 00000000000..c94384a4eb9 --- /dev/null +++ b/v20.2/admin-ui-runtime-dashboard.md @@ -0,0 +1,76 @@ +--- +title: Runtime Dashboard +toc: true +--- + +The **Runtime** dashboard in the CockroachDB Admin UI lets you monitor runtime metrics for you cluster, such as node count, memory usage, and CPU time. To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** on the left-hand navigation bar, and then select **Dashboard** > **Runtime**. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Runtime** dashboard displays the following time series graphs: + +## Live Node Count + +CockroachDB Admin UI Node Count + +In the node view as well as the cluster view, the graph shows the number of live nodes in the cluster. + +A dip in the graph indicates decommissioned nodes, dead nodes, or nodes that are not responding. To troubleshoot the dip in the graph, refer to the [Summary panel](admin-ui-access-and-navigate.html#summary-panel). + +## Memory Usage + +CockroachDB Admin UI Memory Usage + +- In the node view, the graph shows the memory in use for the selected node. + +- In the cluster view, the graph shows the memory in use across all nodes in the cluster. + +On hovering over the graph, the values for the following metrics are displayed: + +Metric | Description +--------|---- +RSS | Total memory in use by CockroachDB. +Go Allocated | Memory allocated by the Go layer. +Go Total | Total memory managed by the Go layer. +CGo Allocated | Memory allocated by the C layer. +CGo Total | Total memory managed by the C layer. + +{{site.data.alerts.callout_info}}If Go Total or CGO Total fluctuates or grows steadily over time, contact us.{{site.data.alerts.end}} + +## CPU Time + +CockroachDB Admin UI CPU Time + + +- In the node view, the graph shows the [CPU time](https://en.wikipedia.org/wiki/CPU_time) used by CockroachDB user and system-level operations for the selected node. +- In the cluster view, the graph shows the [CPU time](https://en.wikipedia.org/wiki/CPU_time) used by CockroachDB user and system-level operations across all nodes in the cluster. + +On hovering over the CPU Time graph, the values for the following metrics are displayed: + +Metric | Description +--------|---- +User CPU Time | Total CPU seconds per second used by the CockroachDB process across all nodes. +Sys CPU Time | Total CPU seconds per second used for CockroachDB system-level operations across all nodes. + +## Clock Offset + +CockroachDB Admin UI Clock Offset + +- In the node view, the graph shows the mean clock offset of the node against the rest of the cluster. +- In the cluster view, the graph shows the mean clock offset of each node against the rest of the cluster. + +## Other graphs + +The **Runtime** dashboard shows other time series graphs that are important for CockroachDB developers: + +- Goroutine Count +- GC Runs +- GC Pause Time + +For monitoring CockroachDB, it is sufficient to use the [**Live Node Count**](#live-node-count), [**Memory Usage**](#memory-usage), [**CPU Time**](#cpu-time), and [**Clock Offset**](#clock-offset) graphs. + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-sql-dashboard.md b/v20.2/admin-ui-sql-dashboard.md new file mode 100644 index 00000000000..3aa569828a7 --- /dev/null +++ b/v20.2/admin-ui-sql-dashboard.md @@ -0,0 +1,87 @@ +--- +title: SQL Dashboard +summary: The SQL dashboard lets you monitor the performance of your SQL queries. +toc: true +--- + +The **SQL** dashboard in the CockroachDB Admin UI lets you monitor the performance of your SQL queries. To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** on the left-hand navigation bar, and then select **Dashboard** > **SQL**. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **SQL** dashboard displays the following time series graphs: + +## SQL Connections + +CockroachDB Admin UI SQL Connections + +- In the node view, the graph shows the number of connections currently open between the client and the selected node. + +- In the cluster view, the graph shows the total number of SQL client connections to all nodes combined. + +## SQL Byte Traffic + +CockroachDB Admin UI SQL Byte Traffic + +The **SQL Byte Traffic** graph helps you correlate SQL query count to byte traffic, especially in bulk data inserts or analytic queries that return data in bulk. + +- In the node view, the graph shows the current byte throughput (bytes/second) between all the currently connected SQL clients and the node. + +- In the cluster view, the graph shows the aggregate client throughput across all nodes. + +## SQL Queries + +CockroachDB Admin UI SQL Queries + +- In the node view, the graph shows the 10-second average of the number of `SELECT`/`INSERT`/`UPDATE`/`DELETE` queries per second issued by SQL clients on the node. + +- In the cluster view, the graph shows the sum of the per-node averages, that is, an aggregate estimation of the current query load over the cluster, assuming the last 10 seconds of activity per node are representative of this load. + +## SQL Query Errors + +CockroachDB Admin UI SQL Query Errors + +- In the node view, the graph shows the 10-second average of the number of SQL statements issued to the node that returned a [planning](architecture/sql-layer.html#sql-parser-planner-executor), [runtime](architecture/sql-layer.html#sql-parser-planner-executor), or [retry error](transactions.html#error-handling). + +- In the cluster view, the graph shows the 10-second average of the number of SQL statements that returned a [planning](architecture/sql-layer.html#sql-parser-planner-executor), [runtime](architecture/sql-layer.html#sql-parser-planner-executor), or [retry error](transactions.html#error-handling) across all nodes. + +## Service Latency: SQL, 99th percentile + +CockroachDB Admin UI Service Latency + +Service latency is calculated as the time between when the cluster receives a query and finishes executing the query. This time does not include returning results to the client. + +- In the node view, the graph displays the 99th [percentile](https://en.wikipedia.org/wiki/Percentile#The_normal_distribution_and_percentiles) of service latency for the selected node. + +- In the cluster view, the graph displays the 99th [percentile](https://en.wikipedia.org/wiki/Percentile#The_normal_distribution_and_percentiles) of service latency for each node in the cluster. + +## Transactions + +CockroachDB Admin UI Transactions + +- In the node view, the graph shows the 10-second average of the number of opened, committed, aborted, and rolled back [transactions](transactions.html) per second issued by SQL clients on the node. + +- In the cluster view, the graph shows the sum of the per-node averages, that is, an aggregate estimation of the current [transactions](transactions.html) load over the cluster, assuming the last 10 seconds of activity per node are representative of this load. + +If the graph shows excessive aborts or rollbacks, it might indicate issues with the SQL queries. In that case, re-examine queries to lower contention. + +Additionally, per-application average transaction times are displayed for each node, at the 90th and 99th percentiles. + +CockroachDB Admin UI Transaction Latencies + +## Other graphs + +The **SQL** dashboard shows other time series graphs that are important for CockroachDB developers: + +- KV Execution Latency +- Active Distributed SQL Queries +- Active Flows for Distributed SQL Queries +- Service Latency: DistSQL +- Schema Changes + +For monitoring CockroachDB, it is sufficient to use the [**SQL Connections**](#sql-connections), [**SQL Byte Traffic**](#sql-byte-traffic), [**SQL Queries**](#sql-queries), [**Service Latency**](#service-latency-sql-99th-percentile), and [**Transactions**](#transactions) graphs. + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/admin-ui-statements-page.md b/v20.2/admin-ui-statements-page.md new file mode 100644 index 00000000000..7d5863d2164 --- /dev/null +++ b/v20.2/admin-ui-statements-page.md @@ -0,0 +1,185 @@ +--- +title: Statements Page +toc: true +--- + +{{site.data.alerts.callout_info}} +On a secure cluster, this area of the Admin UI can only be accessed by an `admin` user. See [Admin UI access](admin-ui-overview.html#admin-ui-access). +{{site.data.alerts.end}} + +The **Statements** page helps you: + +- Identify frequently executed or high latency [SQL statements](sql-statements.html). +- View SQL statement [details](#statement-details-page). +- Download SQL statement [diagnostics](#diagnostics) for troubleshooting. + +To view this page, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and click **Statements** in the left-hand navigation. + +## Search and filter by application + +By default, this page shows SQL statements from all applications running on the cluster, and hides internal CockroachDB queries. + +To filter the statements by [`application_name`](connection-parameters.html#additional-connection-parameters), use the **App** menu. If you haven't set `application_name` in the client connection string, it appears as `unset`. + +CockroachDB's internal queries are only displayed under the `(internal)` app. Queries from the SQL shell are displayed under the `$ cockroach sql` app. + +You can also search for statements using the search bar. + +## Understand the Statements page + +Use this page to identify SQL statements that you may want to [troubleshoot](query-behavior-troubleshooting.html). This might include statements that are experiencing high latencies, multiple [retries](transactions.html#transaction-retries), or execution failures. You can optionally create and retrieve [diagnostics](#diagnostics) for these statements. + +{{site.data.alerts.callout_success}} +If you haven't yet executed any queries in the cluster as a user, this page will initially be blank. +{{site.data.alerts.end}} + +CockroachDB Admin UI Statements Page + +Parameter | Description +-----|------------ +Statement | SQL statement [fingerprint](#sql-statement-fingerprints).

      To view additional details of a SQL statement fingerprint, click this to open the [**Statement Details** page](#statement-details-page). +Txn Type | Type of transaction (implicit or explicit). Explicit transactions refer to statements that are wrapped by [`BEGIN`](begin-transaction.html) and [`COMMIT`](commit-transaction.html) statements by the client. Explicit transactions employ [transactional pipelining](architecture/transaction-layer.html#transaction-pipelining) and therefore report latencies that do not account for replication.

      For statements not in explicit transactions, CockroachDB wraps each statement in individual implicit transactions. +Retries | Cumulative number of [retries](transactions.html#transaction-retries) of statements with this fingerprint within the last hour or specified [time interval](#time-interval). +Execution Count | Cumulative number of executions of statements with this fingerprint within the last hour or specified [time interval](#time-interval).

      The bar indicates the ratio of runtime success (gray) to [retries](transactions.html#transaction-retries) (red) for the SQL statement fingerprint. +Rows Affected | Average number of rows returned while executing statements with this fingerprint within the last hour or specified [time interval](#time-interval).

      The gray bar indicates the mean number of rows returned. The blue bar indicates one standard deviation from the mean. +Latency | Average service latency of statements with this fingerprint within the last hour or specified [time interval](#time-interval). Service latency is the time taken to execute a query once it is received by the cluster. It does not include the time taken to send the query to the cluster or return the result to the client.

      The gray bar indicates the mean latency. The blue bar indicates one standard deviation from the mean. +Diagnostics | Option to activate [diagnostics](#diagnostics) for this fingerprint. If activated, this displays the status of diagnostics collection (`WAITING FOR QUERY`, `READY`, OR `ERROR`). When `READY`, the most recent diagnostics bundle can be downloaded here. Access the full history of diagnostics for the fingerprint in the [**Statement Details**](#statement-details-page) page. + +### Time interval + +By default, the Statements page displays all SQL statements executed within a one-hour time interval. The display is cleared at the end of each interval. You can change the interval with the [`diagnostics.reporting.interval`](cluster-settings.html#settings) cluster setting. + +### SQL statement fingerprints + +The Statements page displays SQL statement *fingerprints*. + +A statement fingerprint represents one or more SQL statements by replacing literal values (e.g., numbers and strings) with underscores (`_`). This can help you quickly identify frequently executed SQL statements and their latencies. + +For multiple SQL statements to be represented by a fingerprint, they must be identical aside from their literal values: + +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES (380, 11, 11098) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES (192, 891, 20) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES (784, 452, 78) + +The above SQL statements have the fingerprint: + +INSERT INTO new_order(product_id, customer_id, no_w_id) VALUES (_, _, _) + +The following statements cannot be represented by the same fingerprint: + +- INSERT INTO orders(product_id, customer_id, transaction_id) VALUES (380, 11, 11098) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES (380, 11, 11098) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES ($1, 11, 11098) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES ($1, $2, 11098) +- INSERT INTO new_order(product_id, customer_id, transaction_id) VALUES ($1, $2, $3) + +It is possible to see the same fingerprint listed multiple times in the following scenarios: + +- Statements with this fingerprint were executed by more than one [`application_name`](show-vars.html#supported-variables). +- Statements with this fingerprint were executed both successfully and unsuccessfully. + +## Statement Details page + +Click on a SQL statement fingerprint to open **Statement Details**. For each statement fingerprint, the details include: + +- [Overview](#overview) +- [Diagnostics](#diagnostics) +- [Logical plan](#logical-plan) +- [Statistics](#execution-stats) + +CockroachDB Admin UI Statements Page + +### Overview + +The **Overview** section displays the SQL statement fingerprint and essential statistics on the right-hand side of the page: + +- **Total Time** is the cumulative time taken to execute statements with this fingerprint within the [specified time interval](#time-interval). +- **Mean Service Latency** is the average service latency of statements with this fingerprint within the [specified time interval](#time-interval). +- **App** displays the name specified by the [`application_name`](show-vars.html#supported-variables) session setting. +- **Transaction Type** displays the type of transaction (implicit or explicit). +- **Distributed execution?** indicates whether the execution was distributed. +- **Used cost-based optimizer?** indicates whether the execution used the [cost-based optimizer](cost-based-optimizer.html). +- **Failed?** indicates whether the execution was successful. + +**Execution Count** displays execution statistics for the SQL statement fingerprint. + +- **First Attempts** is the cumulative number of first attempts at executing statements with this fingerprint within the [specified time interval](#time-interval). +- **Retries** is the cumulative number of [retries](transactions.html#transaction-retries) of statements with this fingerprint within the [specified time interval](#time-interval). +- **Max Retries** is the highest number of retries of a single statement with this fingerprint within the [specified time interval](#time-interval). For example, if three statements with the same fingerprint had to be retried 0, 1, and 5 times, then the Max Retries value for the fingerprint is 5. +- **Total** is the total number of executions of statements with this fingerprint. It is calculated as the sum of first attempts and retries. + +**Rows Affected** displays statistics on rows returned for the SQL statement fingerprint. + +- **Mean Rows** is the average number of rows returned while executing statements with this fingerprint within the [specified time interval](#time-interval). +- **Standard Deviation** is the value of one standard deviation of the mean. + +### Diagnostics + +The **Diagnostics** section of the Statement Details page allows you to activate and view diagnostics for the SQL statement fingerprint. + +When you activate diagnostics for a fingerprint, CockroachDB waits for the next SQL query that matches this fingerprint to be run on any node. On the next match, information about the SQL statement is written to a diagnostics bundle that you can download. This bundle consists of a JSON file that contains a distributed trace of the SQL statement, a physical query plan, execution statistics, and other information about the query. For more details on the contents, see [`EXPLAIN ANALYZE (DEBUG)`](explain-analyze.html#debug-option). + +{{site.data.alerts.callout_success}} +Diagnostics will be collected a maximum of *N* times for a given activated fingerprint where *N* is the number of nodes in your cluster. +{{site.data.alerts.end}} + +CockroachDB Admin UI Statements Page + +- Click the **Activate** button to begin collecting diagnostics for the fingerprint. This will open the list of **Statement diagnostics** with a status next to each activated diagnostic. + - `WAITING FOR QUERY` indicates that a SQL statement matching the fingerprint has not yet been recorded. + - `ERROR` indicates that the attempt at diagnostics collection failed. + - `READY` indicates that the diagnostics have run and can be downloaded. A download link will appear beside the status. +- For any row with a `READY` status, click **Bundle (.zip)** to retrieve the diagnostics. + +After downloading the statement diagnostics, you will have a JSON file that represents transaction events across nodes for the SQL statement. The information collected here can be used to diagnose problematic SQL statements, such as [slow queries](query-behavior-troubleshooting.html#query-is-always-slow). + +We currently recommend that you share the diagnostics with our [support team](support-resources.html), which can help you interpret the results. + +{{site.data.alerts.callout_info}} +This is different from the output of [`SHOW TRACE FOR SESSION`](show-trace.html), which returns messages and timing information for all statements recorded during a session. +{{site.data.alerts.end}} + +Click **All statement diagnostics** to view a complete history of your collected diagnostics, each of which can be downloaded. Although fingerprints are periodically cleared from the Statements page, all diagnostics bundles are preserved. If you need to access diagnostics that were collected for a fingerprint not present in the past [interval](#time-interval), you can find the bundle here. + +### Logical Plan + +The **Logical Plan** section displays CockroachDB's query plan for an [explainable statement](sql-grammar.html#preparable_stmt). You can use this information to optimize the query. For more information about logical plans, see [`EXPLAIN`](explain.html). + +CockroachDB Admin UI Statements Page + +By default, the logical plan for each fingerprint is sampled every 5 minutes. You can change the interval with the [`sql.metrics.statement_details.plan_collection.period`](cluster-settings.html#settings) cluster setting. For example, to change the interval to 2 minutes, run the following [`SET CLUSTER SETTING`](set-cluster-setting.html) command: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.metrics.statement_details.plan_collection.period = '2m0s'; +~~~ + +### Execution Stats + +**Execution Latency by Phase** displays the service latency of statements matching this fingerprint, broken down by [phase](architecture/sql-layer.html#sql-parser-planner-executor) (parse, plan, run, overhead), as well as the overall service latency. The gray bar indicates the mean latency. The blue bar indicates one standard deviation from the mean. + +{{site.data.alerts.callout_success}} +"Overhead" comprises the statements that remain after subtracting parse, plan, and run latencies from the overall latency. These might include fetching table descriptors that were not cached, or other background tasks required to execute the query. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +Service latency can be affected by network latency, which is displayed for your cluster on the [Network Latency](admin-ui-network-latency-page.html) page. +{{site.data.alerts.end}} + +The **Statistics by Node** table provides a breakdown of the number of statements of the selected fingerprint per gateway node. You can use this table to determine whether, for example, you are executing queries on a node that is far from the data you are requesting (see [Make Queries Fast](make-queries-fast.html#cluster-topology)). + +Parameter | Description +-----|------------ +Node | ID of the gateway node. +Retries | Cumulative number of [retries](transactions.html#transaction-retries) of statements with this fingerprint within the last hour or specified [time interval](#time-interval). +Execution Count | Cumulative number of executions of statements with this fingerprint within the last hour or specified [time interval](#time-interval). +Rows Affected | Average number of rows returned while executing statements with this fingerprint within the last hour or specified [time interval](#time-interval).

      The gray bar indicates the mean number of rows returned. The blue bar indicates one standard deviation from the mean. +Latency | Average service latency of statements with this fingerprint within the last hour or specified [time interval](#time-interval). Service latency is the time taken to execute a query once it is received by the cluster. It does not include the time taken to return the result to the client.

      The gray bar indicates the mean latency. The blue bar indicates one standard deviation from the mean. + +## See also + +- [Troubleshoot Query Behavior](query-behavior-troubleshooting.html) +- [Transaction retries](transactions.html#transaction-retries) +- [Make Queries Fast](make-queries-fast.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) \ No newline at end of file diff --git a/v20.2/admin-ui-storage-dashboard.md b/v20.2/admin-ui-storage-dashboard.md new file mode 100644 index 00000000000..5aa3f220d11 --- /dev/null +++ b/v20.2/admin-ui-storage-dashboard.md @@ -0,0 +1,91 @@ +--- +title: Storage Dashboard +summary: The Storage dashboard lets you monitor the storage utilization of your cluster. +toc: true +--- + +The **Storage** dashboard lets you monitor the storage utilization of your cluster. + +To view this dashboard, [access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui), click **Metrics** in the left-hand navigation, and select **Dashboard** > **Storage**. + +{% include {{ page.version.version }}/admin-ui/admin-ui-metrics-navigation.md %} + +The **Storage** dashboard displays the following time series graphs: + +## Capacity + +You can monitor the **Capacity** graph to determine when additional storage is needed (e.g., by [scaling your cluster](cockroach-start.html)). + +CockroachDB Admin UI Capacity graph + +Metric | Description +--------|-------- +**Capacity** | The maximum store size. This value may be set per node using [`--store`](cockroach-start.html#store). If a store size has not been set, this metric displays the actual disk capacity. See [Capacity metrics](#capacity-metrics). +**Available** | The free disk space available to CockroachDB data. +**Used** | The disk space in use by CockroachDB data. This excludes the Cockroach binary, operating system, and other system files. + +### Capacity metrics + +The **Capacity** graph displays disk usage by CockroachDB data in relation to the maximum [store](architecture/storage-layer.html) size, which is determined as follows: + +- If a store size was specified using the [`--store`](cockroach-start.html#store) flag when starting nodes, this value is used as the limit for CockroachDB data. +- If no store size has been explicitly set, the actual disk capacity is used as the limit for CockroachDB data. + +The **available** capacity thus equals the amount of empty disk space, up to the value of the maximum store size. The **used** capacity refers only to disk space occupied by CockroachDB data, which resides in the store directory on each node. + +The disk usage of the Cockroach binary, operating system, and other system files is not shown on the **Capacity** graph. + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} +{{site.data.alerts.end}} + +## Live Bytes + +The **Live Bytes** graph displays the amount of data that can be read by applications and CockroachDB. + +CockroachDB Admin UI Replicas per Store + +Metric | Description +--------|-------- +**Live** | Number of logical bytes stored in live [key-value pairs](architecture/distribution-layer.html#table-data). Live data excludes historical and deleted data. +**System** | Number of physical bytes stored in [system key-value pairs](architecture/distribution-layer.html#meta-ranges). This includes historical and deleted data that has not been [garbage collected](architecture/storage-layer.html#garbage-collection). + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/admin-ui/logical-bytes.md %} +{{site.data.alerts.end}} + +## File Descriptors + +CockroachDB Admin UI File Descriptors + +- In the node view, the graph shows the number of open file descriptors for that node, compared with the file descriptor limit. + +- In the cluster view, the graph shows the number of open file descriptors across all nodes, compared with the file descriptor limit. + +If the Open count is almost equal to the Limit count, increase [File Descriptors](recommended-production-settings.html#file-descriptors-limit). + +{{site.data.alerts.callout_info}} +If you are running multiple nodes on a single machine (not recommended), the actual number of open file descriptors are considered open on each node. Thus the limit count value displayed on the Admin UI is the actual value of open file descriptors multiplied by the number of nodes, compared with the file descriptor limit. +{{site.data.alerts.end}} + +For Windows systems, you can ignore the File Descriptors graph because the concept of file descriptors is not applicable to Windows. + +## Other graphs + +The **Storage** dashboard shows other time series graphs that are important for CockroachDB developers: + +- Log Commit Latency +- Command Commit Latency +- Read Amplification +- SSTables +- Compactions/Flushes +- Time Series Writes +- Time Series Bytes Written + +For monitoring CockroachDB, it is sufficient to use the [**Capacity**](#capacity) and [**File Descriptors**](#file-descriptors) graphs. + +## See also + +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) +- [Raw Status Endpoints](monitoring-and-alerting.html#raw-status-endpoints) diff --git a/v20.2/advanced-client-side-transaction-retries.md b/v20.2/advanced-client-side-transaction-retries.md new file mode 100644 index 00000000000..740dec310c8 --- /dev/null +++ b/v20.2/advanced-client-side-transaction-retries.md @@ -0,0 +1,79 @@ +--- +title: Advanced Client-side Transaction Retries +summary: Advanced client-side transaction retry features for library authors +toc: true +--- + +This page has instructions for authors of [database drivers and ORMs](install-client-drivers.html) who would like to implement client-side retries in their database driver or ORM for maximum efficiency and ease of use by application developers. + +{{site.data.alerts.callout_info}} +If you are an application developer who needs to implement an application-level retry loop, see the [Client-side intervention example](transactions.html#client-side-intervention-example). +{{site.data.alerts.end}} + +## Overview + +To improve the performance of transactions that fail due to [contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention), CockroachDB includes a set of statements (listed below) that let you retry those transactions. Retrying transactions using these statements has the following benefits: + +1. When you use savepoints, you "hold your place in line" between attempts. Without savepoints, you're starting from scratch every time. +2. Transactions increase their priority each time they're retried, increasing the likelihood they will succeed. This has a lesser effect than #1. + +## How transaction retries work + +A retryable transaction goes through the process described below, which maps to the following SQL statements: + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; -- #1 +> SAVEPOINT cockroach_restart; -- #2 +-- ... various transaction statements ... -- #3 +> RELEASE SAVEPOINT cockroach_restart; -- #5 (Or #4, ROLLBACK, in case of retry error) +> COMMIT; +~~~ + +1. The transaction starts with the [`BEGIN`](begin-transaction.html) statement. + +2. The [`SAVEPOINT`](savepoint.html) statement shown here is a [retry savepoint](#retry-savepoints); that is, it declares the intention to retry the transaction in the case of contention errors. It must be executed after [`BEGIN`](begin-transaction.html), but before the first statement that manipulates a database. Although [nested transactions](savepoint.html#savepoints-for-nested-transactions) are supported in versions of CockroachDB 20.1 and later, a retry savepoint must be the outermost savepoint in a transaction. + +3. The statements in the transaction are executed. + +4. If a statement returns a retry error (identified via the `40001` error code or `"retry transaction"` string at the start of the error message), you can issue the [`ROLLBACK TO SAVEPOINT`](rollback-transaction.html) statement to restart the transaction and increase the transaction's priority. Alternately, the original [`SAVEPOINT`](savepoint.html) statement can be reissued to restart the transaction. + + You must now issue the statements in the transaction again. + + In cases where you do not want the application to retry the transaction, you can issue [`ROLLBACK`](rollback-transaction.html) at this point. Any other statements will be rejected by the server, as is generally the case after an error has been encountered and the transaction has not been closed. + +5. Once the transaction executes all statements without encountering contention errors, execute [`RELEASE SAVEPOINT`](release-savepoint.html) to commit the changes. If this succeeds, all changes made by the transaction become visible to subsequent transactions and are guaranteed to be durable if a crash occurs. + + In some cases, the [`RELEASE SAVEPOINT`](release-savepoint.html) statement itself can fail with a retry error, mainly because transactions in CockroachDB only realize that they need to be restarted when they attempt to commit. If this happens, the retry error is handled as described in step 4. + +## Retry savepoints + +A savepoint defined with the name `cockroach_restart` is a "retry savepoint" and is used to implement advanced client-side transaction retries. A retry savepoint differs from a [savepoint for nested transactions](savepoint.html#savepoints-for-nested-transactions) as follows: + +- It must be the outermost savepoint in the transaction. +- After a successful [`RELEASE`](release-savepoint.html), a retry savepoint does not allow further use of the transaction. The next statement must be a [`COMMIT`](commit-transaction.html). +- It cannot be nested. Issuing `SAVEPOINT cockroach_restart` two times in a row only creates a single savepoint marker (this can be verified with [`SHOW SAVEPOINT STATUS`](show-savepoint-status.html)). Issuing `SAVEPOINT cockroach_restart` after `ROLLBACK TO SAVEPOINT cockroach_restart` reuses the marker instead of creating a new one. + +Note that you can [customize the retry savepoint name](#customizing-the-retry-savepoint-name) to something other than `cockroach_restart` with a session variable if you need to. + +## Customizing the retry savepoint name + +{% include {{ page.version.version }}/misc/customizing-the-savepoint-name.md %} + +## Examples + +For examples showing how to use [`SAVEPOINT`](savepoint.html) and the other statements described on this page to implement library support for a programming language, see the following: + +- [Build a Java app with CockroachDB](build-a-java-app-with-cockroachdb.html), in particular the logic in the `runSQL` method. +- The source code of the [sqlalchemy-cockroachdb](https://github.com/cockroachdb/sqlalchemy-cockroachdb) adapter for SQLAlchemy. + +## See also + +- [Transactions](transactions.html) +- [`BEGIN`](begin-transaction.html) +- [`COMMIT`](commit-transaction.html) +- [`ROLLBACK`](rollback-transaction.html) +- [`SAVEPOINT`](savepoint.html) +- [`RELEASE SAVEPOINT`](release-savepoint.html) +- [`SHOW`](show-vars.html) +- [CockroachDB Architecture: Transaction Layer](architecture/transaction-layer.html) diff --git a/v20.2/alter-column.md b/v20.2/alter-column.md new file mode 100644 index 00000000000..f44c1707ae9 --- /dev/null +++ b/v20.2/alter-column.md @@ -0,0 +1,89 @@ +--- +title: ALTER COLUMN +summary: Use the ALTER COLUMN statement to set, change, or drop a column's DEFAULT constraint or to drop the NOT NULL constraint. +toc: true +--- + +The `ALTER COLUMN` [statement](sql-statements.html) is part of `ALTER TABLE` and can be used to: +- Set, change, or drop a column's [`DEFAULT` constraint](default-value.html) +- Set or drop a column's [`NOT NULL` constraint](not-null.html) + +{{site.data.alerts.callout_info}} +To manage other constraints, see [`ADD CONSTRAINT`](add-constraint.html) and [`DROP CONSTRAINT`](drop-constraint.html). +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
      +{% include {{ page.version.version }}/sql/diagrams/alter_column.html %} +
      + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + +| Parameter | Description | +|-----------|-------------| +| `table_name` | The name of the table with the column you want to modify. | +| `column_name` | The name of the column you want to modify. | +| `a_expr` | The new [Default Value](default-value.html) you want to use. | + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Set or change a `DEFAULT` value + +Setting the [`DEFAULT` value constraint](default-value.html) inserts the value when data's written to the table without explicitly defining the value for the column. If the column already has a `DEFAULT` value set, you can use this statement to change it. + +The below example inserts the Boolean value `true` whenever you inserted data to the `subscriptions` table without defining a value for the `newsletter` column. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE subscriptions ALTER COLUMN newsletter SET DEFAULT true; +~~~ + +### Remove `DEFAULT` constraint + +If the column has a defined [`DEFAULT` value](default-value.html), you can remove the constraint, which means the column will no longer insert a value by default if one is not explicitly defined for the column. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE subscriptions ALTER COLUMN newsletter DROP DEFAULT; +~~~ + +### Set `NOT NULL` constraint + +Setting the [`NOT NULL` constraint](not-null.html) specifies that the column cannot contain `NULL` values. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE subscriptions ALTER COLUMN newsletter SET NOT NULL; +~~~ + +### Remove `NOT NULL` constraint + +If the column has the [`NOT NULL` constraint](not-null.html) applied to it, you can remove the constraint, which means the column becomes optional and can have *NULL* values written into it. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE subscriptions ALTER COLUMN newsletter DROP NOT NULL; +~~~ + +### Convert a computed column into a regular column + +{% include {{ page.version.version }}/computed-columns/convert-computed-column.md %} + +## See also + +- [Constraints](constraints.html) +- [`ADD CONSTRAINT`](add-constraint.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`ALTER TABLE`](alter-table.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/alter-database.md b/v20.2/alter-database.md new file mode 100644 index 00000000000..5492fea57a2 --- /dev/null +++ b/v20.2/alter-database.md @@ -0,0 +1,20 @@ +--- +title: ALTER DATABASE +summary: Use the ALTER DATABASE statement to change an existing database. +toc: false +--- + +The `ALTER DATABASE` [statement](sql-statements.html) applies a schema change to a database. For information on using `ALTER DATABASE`, see the pages for its relevant [subcommands](#subcommands). + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Subcommands + +Subcommand | Description +-----------|------------ +[`CONFIGURE ZONE`](configure-zone.html) | [Configure replication zones](configure-replication-zones.html) for a database. +[`RENAME`](rename-database.html) | Change the name of a database. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} diff --git a/v20.2/alter-index.md b/v20.2/alter-index.md new file mode 100644 index 00000000000..e14615f48c4 --- /dev/null +++ b/v20.2/alter-index.md @@ -0,0 +1,23 @@ +--- +title: ALTER INDEX +summary: Use the ALTER INDEX statement to change an existing index. +toc: true +--- + +The `ALTER INDEX` [statement](sql-statements.html) applies a schema change to an index. For information on using `ALTER INDEX`, see the pages for its relevant [subcommands](#subcommands). + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Subcommands + +Subcommand | Description +-----------|------------ +[`CONFIGURE ZONE`](configure-zone.html) | [Configure replication zones](configure-replication-zones.html) for an index. +[`PARTITION BY`](partition-by.html) | Partition, re-partition, or un-partition an index. ([Enterprise-only](enterprise-licensing.html)). +[`RENAME`](rename-index.html) | Change the name of an index. +[`SPLIT AT`](split-at.html) | Force a range split at the specified row in the index. +[`UNSPLIT AT`](unsplit-at.html) | Remove a range split enforcement at a specified row in the index. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} diff --git a/v20.2/alter-partition.md b/v20.2/alter-partition.md new file mode 100644 index 00000000000..01e59232a62 --- /dev/null +++ b/v20.2/alter-partition.md @@ -0,0 +1,11 @@ +--- +title: ALTER PARTITION +summary: Use the ALTER PARTITION statement to configure the replication zone for a partition. +toc: true +--- + +The `ALTER PARTITION` [statement](sql-statements.html) is used to configure replication zones for partitions. See the [`CONFIGURE ZONE`](configure-zone.html) subcommand for more details. + +{{site.data.alerts.callout_info}} +[Partitioning](partitioning.html) is an [enterprise-only](enterprise-licensing.html) feature. +{{site.data.alerts.end}} diff --git a/v20.2/alter-primary-key.md b/v20.2/alter-primary-key.md new file mode 100644 index 00000000000..03f88b43a39 --- /dev/null +++ b/v20.2/alter-primary-key.md @@ -0,0 +1,238 @@ +--- +title: ALTER PRIMARY KEY +summary: Use the ALTER PRIMARY KEY statement to change the primary key of a table. +toc: true +--- + + The `ALTER PRIMARY KEY` [statement](sql-statements.html) is a subcommand of [`ALTER TABLE`](alter-table.html) that can be used to change the [primary key](primary-key.html) of a table. + +## Details + +- You cannot change the primary key of a table that is currently undergoing a primary key change, or any other [schema change](online-schema-changes.html). + +- `ALTER PRIMARY KEY` might need to rewrite multiple indexes, which can make it an expensive operation. + +- When you change a primary key with `ALTER PRIMARY KEY`, the old primary key index becomes a [`UNIQUE`](unique.html) secondary index. This helps optimize the performance of queries that still filter on the old primary key column. + +- `ALTER PRIMARY KEY` does not alter the [partitions](partitioning.html) on a table or its indexes, even if a partition is defined on [a column in the original primary key](partitioning.html#partition-using-primary-key). If you alter the primary key of a partitioned table, you must update the table partition accordingly. + +- The secondary index created by `ALTER PRIMARY KEY` will not be partitioned, even if a partition is defined on [a column in the original primary key](partitioning.html#partition-using-primary-key). To ensure that the table is partitioned correctly, you must create a partition on the secondary index, or drop the secondary index. + +{{site.data.alerts.callout_success}} +To change an existing primary key without creating a secondary index from that primary key, use [`DROP CONSTRAINT ... PRIMARY KEY`/`ADD CONSTRAINT ... PRIMARY KEY`](add-constraint.html#changing-primary-keys-with-add-constraint-primary-key). For examples, see the [`ADD CONSTRAINT`](add-constraint.html#examples) and [`DROP CONSTRAINT`](drop-constraint.html#examples) pages. +{{site.data.alerts.end}} + +## Synopsis + +
      +{% include {{ page.version.version }}/sql/diagrams/alter_primary_key.html %} +
      + +## Parameters + + Parameter | Description +-----------|------------- + `table_name` | The name of the table with the primary key that you want to modify. + `index_params` | The name of the column(s) that you want to use for the primary key. These columns replace the current primary key column(s). + `opt_interleave` | You can potentially optimize query performance by [interleaving tables](interleave-in-parent.html), which changes how CockroachDB stores your data.
      {{site.data.alerts.callout_info}}[Hash-sharded indexes](indexes.html#hash-sharded-indexes) cannot be interleaved.{{site.data.alerts.end}} + `USING HASH WITH BUCKET COUNT` | Creates a [hash-sharded index](indexes.html#hash-sharded-indexes) with `n_buckets` number of buckets.
      {{site.data.alerts.callout_info}}To enable hash-sharded indexes, set the `experimental_enable_hash_sharded_indexes` [session variable](set-vars.html) to `on`.{{site.data.alerts.end}} + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on a table to alter its primary key. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Alter a single-column primary key + +Suppose that you are storing the data for users of your application in a table called `users`, defined by the following `CREATE TABLE` statement: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + name STRING PRIMARY KEY, + email STRING +); +~~~ + +The primary key of this table is on the `name` column. This is a poor choice, as some users likely have the same name, and all primary keys enforce a `UNIQUE` constraint on row values of the primary key column. Per our [best practices](performance-best-practices-overview.html#use-uuid-to-generate-unique-ids), you should instead use a `UUID` for single-column primary keys, and populate the rows of the table with generated, unique values. + +You can add a column and change the primary key with a couple of `ALTER TABLE` statements: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD COLUMN id UUID NOT NULL DEFAULT gen_random_uuid(); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ALTER PRIMARY KEY USING COLUMNS (id); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+-------------------------------------------------- + users | CREATE TABLE users ( + | name STRING NOT NULL, + | email STRING NULL, + | id UUID NOT NULL DEFAULT gen_random_uuid(), + | CONSTRAINT "primary" PRIMARY KEY (id ASC), + | UNIQUE INDEX users_name_key (name ASC), + | FAMILY "primary" (name, email, id) + | ) +(1 row) +~~~ + +Note that the old primary key index becomes a secondary index, in this case, `users_name_key`. If you do not want the old primary key to become a secondary index when changing a primary key, you can use [`DROP CONSTRAINT`](drop-constraint.html)/[`ADD CONSTRAINT`](add-constraint.html) instead. + +### Make a single-column primary key composite for geo-partitioning + +Suppose that you are storing the data for users of your application in a table called `users`, defined by the following `CREATE TABLE` statement: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email STRING, + name STRING, + INDEX users_name_idx (name) +); +~~~ + +Now suppose that you want to expand your business from a single region into multiple regions. After you [deploy your application in multiple regions](topology-patterns.html), you consider [geo-partitioning your data](topology-geo-partitioned-replicas.html) to minimize latency and optimize performance. In order to geo-partition the `user` database, you need to add a column specifying the location of the data (e.g., `region`): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD COLUMN region STRING NOT NULL; +~~~ + +When you geo-partition a database, you [partition the database on a primary key column](partitioning.html#partition-using-primary-key). The primary key of this table is still on `id`. Change the primary key to be composite, on `region` and `id`: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ALTER PRIMARY KEY USING COLUMNS (region, id); +~~~ +{{site.data.alerts.callout_info}} +The order of the primary key columns is important when geo-partitioning. For performance, always place the partition column first. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL DEFAULT gen_random_uuid(), + | email STRING NULL, + | name STRING NULL, + | region STRING NOT NULL, + | CONSTRAINT "primary" PRIMARY KEY (region ASC, id ASC), + | UNIQUE INDEX users_id_key (id ASC), + | INDEX users_name_idx (name ASC), + | FAMILY "primary" (id, email, name, region) + | ) +(1 row) +~~~ + +Note that the old primary key index on `id` is now the secondary index `users_id_key`. + +With the new primary key on `region` and `id`, the table is ready to be [geo-partitioned](topology-geo-partitioned-replicas.html): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users PARTITION BY LIST (region) ( + PARTITION us_west VALUES IN ('us_west'), + PARTITION us_east VALUES IN ('us_east') + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER PARTITION us_west OF INDEX users@primary + CONFIGURE ZONE USING constraints = '[+region=us-west1]'; + ALTER PARTITION us_east OF INDEX users@primary + CONFIGURE ZONE USING constraints = '[+region=us-east1]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW PARTITIONS FROM TABLE users; +~~~ + +~~~ + database_name | table_name | partition_name | parent_partition | column_names | index_name | partition_value | zone_config | full_zone_config +----------------+------------+----------------+------------------+--------------+---------------+-----------------+------------------------------------+-------------------------------------- + movr | users | us_west | NULL | region | users@primary | ('us_west') | constraints = '[+region=us-west1]' | range_min_bytes = 134217728, + | | | | | | | | range_max_bytes = 536870912, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-west1]', + | | | | | | | | lease_preferences = '[]' + movr | users | us_east | NULL | region | users@primary | ('us_east') | constraints = '[+region=us-east1]' | range_min_bytes = 134217728, + | | | | | | | | range_max_bytes = 536870912, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-east1]', + | | | | | | | | lease_preferences = '[]' +(2 rows) +~~~ + +The table is now geo-partitioned on the `region` column. + +You now need to geo-partition any secondary indexes in the table. In order to geo-partition an index, the index must be prefixed by a column that can be used as a partitioning identifier (in this case, `region`). Currently, neither of the secondary indexes (i.e., `users_id_key` and `users_name_idx`) are prefixed by the `region` column, so they can't be meaningfully geo-partitioned. Any secondary indexes that you want to keep must be dropped, recreated, and then partitioned. + +Start by dropping both indexes: + +{% include copy-clipboard.html %} +~~~ sql +> DROP INDEX users_id_key CASCADE; + DROP INDEX users_name_idx CASCADE; +~~~ + +You don't need to recreate the index on `id` with `region`. Both columns are already indexed by the new primary key. + +Add `region` to the index on `name`: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users(region, name); +~~~ + +Then geo-partition the index: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER INDEX users_region_name_idx PARTITION BY LIST (region) ( + PARTITION us_west VALUES IN ('us_west'), + PARTITION us_east VALUES IN ('us_east') + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER PARTITION us_west OF INDEX users@users_region_name_idx + CONFIGURE ZONE USING constraints = '[+region=us-west1]'; + ALTER PARTITION us_east OF INDEX users@users_region_name_idx + CONFIGURE ZONE USING constraints = '[+region=us-east1]'; +~~~ + + +## See also + +- [Constraints](constraints.html) +- [`ADD CONSTRAINT`](add-constraint.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`ALTER TABLE`](alter-table.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/alter-range.md b/v20.2/alter-range.md new file mode 100644 index 00000000000..664d8b7f1d0 --- /dev/null +++ b/v20.2/alter-range.md @@ -0,0 +1,7 @@ +--- +title: ALTER RANGE +summary: Use the ALTER RANGE statement to configure the replication zone for a system range. +toc: true +--- + +The `ALTER RANGE` [statement](sql-statements.html) is used to configure replication zones for system ranges. See the [`CONFIGURE ZONE`](configure-zone.html) subcommand for more details. diff --git a/v20.2/alter-role.md b/v20.2/alter-role.md new file mode 100644 index 00000000000..5e6f672fea2 --- /dev/null +++ b/v20.2/alter-role.md @@ -0,0 +1,183 @@ +--- +title: ALTER ROLE +summary: The ALTER ROLE statement can be used to add or change a role's password. +toc: true +--- + + The `ALTER ROLE` [statement](sql-statements.html) can be used to add, change, or remove a [role's](create-role.html) password and to change the login privileges for a role. + +{{site.data.alerts.callout_info}} +Since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `ALTER ROLE` is now an alias for [`ALTER USER`](alter-user.html). +{{site.data.alerts.end}} + +## Considerations + +- Password creation and alteration is supported only in secure clusters. + +## Required privileges + + To alter other roles, the role must have the [`CREATEROLE`](create-role.html#allow-the-role-to-create-other-roles) parameter set. + +## Synopsis + +
      {% include {{ page.version.version }}/sql/diagrams/alter_role.html %}
      + +## Parameters + + + +Parameter | Description +----------|------------- +`name` | The name of the role whose password you want to create or add. +`password` | Let the role [authenticate their access to a secure cluster](authentication.html#client-authentication) using this new password. Passwords should be entered as a [string literal](sql-constants.html#string-literals). For compatibility with PostgreSQL, a password can also be entered as an [identifier](#change-password-using-an-identifier).

      To prevent a role from using [password authentication](authentication.html#client-authentication) and to mandate [certificate-based client authentication](authentication.html#client-authentication), [set the password as `NULL`](#prevent-a-role-from-using-password-authentication). +`VALID UNTIL` | The date and time (in the [`timestamp`](timestamp.html) format) after which the password is not valid. +`LOGIN`/`NOLOGIN` | The `LOGIN` parameter allows a role to login with one of the [client authentication methods](authentication.html#client-authentication). [Setting the parameter to `NOLOGIN`](#change-login-privileges-for-a-role) prevents the role from logging in using any authentication method. +`CREATEROLE`/`NOCREATEROLE` | Allow or disallow the role to create, alter, and drop other roles.

      By default, the parameter is set to `NOCREATEROLE` for all non-admin and non-root roles. + +## Examples + +### Change password using a string literal + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl WITH PASSWORD 'ilov3beefjerky'; +~~~ +~~~ +ALTER ROLE 1 +~~~ + +### Change password using an identifier + +The following statement changes the password to `ilov3beefjerky`, as above: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl WITH PASSWORD ilov3beefjerky; +~~~ + +This is equivalent to the example in the previous section because the password contains only lowercase characters. + +In contrast, the following statement changes the password to `thereisnotomorrow`, even though the password in the syntax contains capitals, because identifiers are normalized automatically: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl WITH PASSWORD ThereIsNoTomorrow; +~~~ + +To preserve case in a password specified using identifier syntax, use double quotes: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl WITH PASSWORD "ThereIsNoTomorrow"; +~~~ + +### Set password validity + +The following statement sets the date and time after which the password is not valid: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl VALID UNTIL '2021-01-01'; +~~~ + +### Prevent a role from using password authentication + +The following statement prevents the role from using password authentication and mandates certificate-based client authentication: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl WITH PASSWORD NULL; +~~~ + +### Change login privileges for a role + +The following statement prevents the role from logging in with any [client authentication method](authentication.html#client-authentication): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl NOLOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ROLES; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | NOLOGIN | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +The following statement allows the role to log in with one of the client authentication methods: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl LOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ROLES; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +### Allow the role to create other roles + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ROLES; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER ROLE carl with CREATEROLE; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ROLES; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | CREATEROLE | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + + +## See also + +- [`DROP ROLE`](drop-role.html) +- [`SHOW ROLES`](show-roles.html) +- [`GRANT `](grant.html) +- [`SHOW GRANTS`](show-grants.html) +- [Create Security Certificates](cockroach-cert.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/alter-sequence.md b/v20.2/alter-sequence.md new file mode 100644 index 00000000000..54ef0f2fbf5 --- /dev/null +++ b/v20.2/alter-sequence.md @@ -0,0 +1,119 @@ +--- +title: ALTER SEQUENCE +summary: Use the ALTER SEQUENCE statement to change the name, increment values, and other settings of a sequence. +toc: true +--- + +The `ALTER SEQUENCE` [statement](sql-statements.html) [changes the name](rename-sequence.html), increment values, and other settings of a sequence. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database. + +## Synopsis + +
      {% include {{ page.version.version }}/sql/diagrams/alter_sequence_options.html %}
      + +## Parameters + + + + Parameter | Description +-----------|------------ +`IF EXISTS` | Modify the sequence only if it exists; if it does not exist, do not return an error. +`sequence_name` | The name of the sequence you want to modify. +`INCREMENT` | The new value by which the sequence is incremented. A negative number creates a descending sequence. A positive number creates an ascending sequence. +`MINVALUE` | The new minimum value of the sequence.

      **Default:** `1` +`MAXVALUE` | The new maximum value of the sequence.

      **Default:** `9223372036854775807` +`START` | The value the sequence starts at if you `RESTART` or if the sequence hits the `MAXVALUE` and `CYCLE` is set.

      `RESTART` and `CYCLE` are not implemented yet. +`CYCLE` | The sequence will wrap around when the sequence value hits the maximum or minimum value. If `NO CYCLE` is set, the sequence will not wrap. +`OWNED BY column_name` | Associates the sequence to a particular column. If that column or its parent table is dropped, the sequence will also be dropped.

      Specifying an owner column with `OWNED BY` replaces any existing owner column on the sequence. To remove existing column ownership on the sequence and make the column free-standing, specify `OWNED BY NONE`.

      **Default:** `NONE` + +## Examples + +### Change the increment value of a sequence + +In this example, we're going to change the increment value of a sequence from its current state (i.e., `1`) to `2`. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER SEQUENCE customer_seq INCREMENT 2; +~~~ + +Next, we'll add another record to the table and check that the new record adheres to the new sequence. + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customer_list (customer, address) VALUES ('Marie', '333 Ocean Ave'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customer_list; +~~~ +~~~ ++----+----------+--------------------+ +| id | customer | address | ++----+----------+--------------------+ +| 1 | Lauren | 123 Main Street | +| 2 | Jesse | 456 Broad Ave | +| 3 | Amruta | 9876 Green Parkway | +| 5 | Marie | 333 Ocean Ave | ++----+----------+--------------------+ +~~~ + +### Set the next value of a sequence + +In this example, we're going to change the next value of the example sequence (`customer_seq`). Currently, the next value will be `7` (i.e., `5` + `INCREMENT 2`). We will change the next value to `20`. + +{{site.data.alerts.callout_info}}You cannot set a value outside the MAXVALUE or MINVALUE of the sequence. {{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SELECT setval('customer_seq', 20, false); +~~~ +~~~ ++--------+ +| setval | ++--------+ +| 20 | ++--------+ +~~~ + +{{site.data.alerts.callout_info}} +The `setval('seq_name', value, is_called)` function in CockroachDB SQL mimics the `setval()` function in PostgreSQL, but it does not store the `is_called` flag. Instead, it sets the value to `val - increment` for `false` or `val` for `true`. +{{site.data.alerts.end}} + +Let's add another record to the table to check that the new record adheres to the new next value. + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customer_list (customer, address) VALUES ('Lola', '333 Schermerhorn'); +~~~ +~~~ ++----+----------+--------------------+ +| id | customer | address | ++----+----------+--------------------+ +| 1 | Lauren | 123 Main Street | +| 2 | Jesse | 456 Broad Ave | +| 3 | Amruta | 9876 Green Parkway | +| 5 | Marie | 333 Ocean Ave | +| 20 | Lola | 333 Schermerhorn | ++----+----------+--------------------+ +~~~ + +## See also + +- [`RENAME SEQUENCE`](rename-sequence.html) +- [`CREATE SEQUENCE`](create-sequence.html) +- [`DROP SEQUENCE`](drop-sequence.html) +- [`SHOW SEQUENCES`](show-sequences.html) +- [Functions and Operators](functions-and-operators.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/alter-table.md b/v20.2/alter-table.md new file mode 100644 index 00000000000..fa0bc97ebd2 --- /dev/null +++ b/v20.2/alter-table.md @@ -0,0 +1,38 @@ +--- +title: ALTER TABLE +summary: Use the ALTER TABLE statement to change the schema of a table. +toc: true +--- + +The `ALTER TABLE` [statement](sql-statements.html) applies a schema change to a table. For information on using `ALTER TABLE`, see the pages for its relevant [subcommands](#subcommands). + +{% include {{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Subcommands + +{{site.data.alerts.callout_success}} +Some subcommands can be used in combination in a single `ALTER TABLE` statement. For example, you can [atomically rename a column and add a new column with the old name of the existing column](rename-column.html#add-and-rename-columns-atomically). +{{site.data.alerts.end}} + +Subcommand | Description | Can combine with other subcommands? +-----------|-------------|------------------------------------ +[`ADD COLUMN`](add-column.html) | Add columns to tables. | Yes +[`ADD CONSTRAINT`](add-constraint.html) | Add constraints to columns. | Yes +[`ALTER COLUMN`](alter-column.html) | Change or drop a column's [`DEFAULT` constraint](default-value.html) or [`NOT NULL` constraint](not-null.html). | Yes +[`ALTER PRIMARY KEY`](alter-primary-key.html) | Change the [primary key](primary-key.html) of a table. | Yes +[`ALTER TYPE`](alter-type.html) | Change a column's [data type](data-types.html). | Yes +[`CONFIGURE ZONE`](configure-zone.html) | [Configure replication zones](configure-replication-zones.html) for a table. | No +[`DROP COLUMN`](drop-column.html) | Remove columns from tables. | Yes +[`DROP CONSTRAINT`](drop-constraint.html) | Remove constraints from columns. | Yes +[`EXPERIMENTAL_AUDIT`](experimental-audit.html) | Enable per-table audit logs. | Yes +[`PARTITION BY`](partition-by.html) | Partition, re-partition, or un-partition a table ([Enterprise-only](enterprise-licensing.html)). | Yes +[`RENAME COLUMN`](rename-column.html) | Change the names of columns. | Yes +[`RENAME CONSTRAINT`](rename-constraint.html) | Change constraints columns. | Yes +[`RENAME TABLE`](rename-table.html) | Change the names of tables. | No +[`SPLIT AT`](split-at.html) | Force a range split at the specified row in the table. | No +[`UNSPLIT AT`](unsplit-at.html) | Remove a range split enforcement at a specified row in the table. | No +[`VALIDATE CONSTRAINT`](validate-constraint.html) | Check whether values in a column match a [constraint](constraints.html) on the column. | Yes + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} diff --git a/v20.2/alter-type.md b/v20.2/alter-type.md new file mode 100644 index 00000000000..b8ddfbd7eef --- /dev/null +++ b/v20.2/alter-type.md @@ -0,0 +1,85 @@ +--- +title: ALTER TYPE +summary: Use the ALTER TYPE statement to change a column's data type. +toc: true +--- + +The `ALTER TYPE` [statement](sql-statements.html) is part of [`ALTER TABLE`](alter-table.html) and changes a column's [data type](data-types.html). + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Considerations + +You can use the `ALTER TYPE` subcommand if the following conditions are met: + +- On-disk representation of the column remains unchanged. For example, you cannot change the column data type from `STRING` to an `INT`, even if the string is just a number. +- The existing data remains valid. For example, you can change the column data type from `STRING[10]` to `STRING[20]`, but not to `STRING [5]` since that will invalidate the existing data. + +## Synopsis + +
      +{% include {{ page.version.version }}/sql/diagrams/alter_type.html %} +
      + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + +| Parameter | Description +|-----------|------------- +| `table_name` | The name of the table with the column whose data type you want to change. +| `column_name` | The name of the column whose data type you want to change. +| `typename` | The new [data type](data-types.html) you want to use. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Success scenario + +The [TPC-C](performance-benchmarking-with-tpc-c-1k-warehouses.html) database has a `customer` table with a column `c_credit_lim DECIMAL (10,2)`. Suppose you want to change the data type to `DECIMAL (12,2)`: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE customer ALTER c_credit_lim type DECIMAL (12,2); +~~~ + +~~~ +ALTER TABLE + +Time: 80.814044ms +~~~ + +### Error scenarios + +Changing a column data type from `DECIMAL` to `INT` would change the on-disk representation of the column. Therefore, attempting to do so results in an error: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE customer ALTER c_credit_lim type INT; +~~~ + +~~~ +pq: type conversion not yet implemented +~~~ + +Changing a column data type from `DECIMAL(12,2)` to `DECIMAL (8,2)` would invalidate the existing data. Therefore, attempting to do so results in an error: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE customer ALTER c_credit_lim type DECIMAL (8,2); +~~~ + +~~~ +pq: type conversion not yet implemented +~~~ + +## See also + +- [`ALTER TABLE`](alter-table.html) +- [Other SQL Statements](sql-statements.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/alter-user.md b/v20.2/alter-user.md new file mode 100644 index 00000000000..dd177b4f920 --- /dev/null +++ b/v20.2/alter-user.md @@ -0,0 +1,147 @@ +--- +title: ALTER USER +summary: The ALTER USER statement can be used to add or change a user's password. +toc: true +--- + +The `ALTER USER` [statement](sql-statements.html) can be used to add, change, or remove a [user's](create-user.html) password and to change the login privileges for a user. + +{{site.data.alerts.callout_info}} + Since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `ALTER USER` is now an alias for [`ALTER ROLE`](alter-role.html). +{{site.data.alerts.end}} + +## Considerations + +- Password creation and alteration is supported only in secure clusters. + +## Required privileges + + To alter other users, the user must have the [`CREATEROLE`](create-user.html#allow-the-user-to-create-other-users) parameter set. + +## Synopsis + +
      {% include {{ page.version.version }}/sql/diagrams/alter_user_password.html %}
      + +## Parameters + + + +Parameter | Description +----------|------------- +`name` | The name of the user whose password you want to create or add. +`password` | Let the user [authenticate their access to a secure cluster](authentication.html#client-authentication) using this new password. Passwords should be entered as a [string literal](sql-constants.html#string-literals). For compatibility with PostgreSQL, a password can also be entered as an [identifier](#change-password-using-an-identifier).

      To prevent a user from using [password authentication](authentication.html#client-authentication) and to mandate [certificate-based client authentication](authentication.html#client-authentication), [set the password as `NULL`](#prevent-a-user-from-using-password-authentication). +`VALID UNTIL` | The date and time (in the [`timestamp`](timestamp.html) format) after which the password is not valid. +`LOGIN`/`NOLOGIN` | The `LOGIN` parameter allows a user to login with one of the [client authentication methods](authentication.html#client-authentication). [Setting the parameter to `NOLOGIN`](#change-login-privileges-for-a-user) prevents the user from logging in using any authentication method. +`CREATEROLE`/`NOCREATEROLE` | Allow or disallow the user to create, alter, and drop other users.

      By default, the parameter is set to `NOCREATEROLE` for all non-admin and non-root users. + +## Examples + +### Change password using a string literal + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl WITH PASSWORD 'ilov3beefjerky'; +~~~ +~~~ +ALTER USER 1 +~~~ + +### Change password using an identifier + +The following statement changes the password to `ilov3beefjerky`, as above: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl WITH PASSWORD ilov3beefjerky; +~~~ + +This is equivalent to the example in the previous section because the password contains only lowercase characters. + +In contrast, the following statement changes the password to `thereisnotomorrow`, even though the password in the syntax contains capitals, because identifiers are normalized automatically: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl WITH PASSWORD ThereIsNoTomorrow; +~~~ + +To preserve case in a password specified using identifier syntax, use double quotes: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl WITH PASSWORD "ThereIsNoTomorrow"; +~~~ + +### Set password validity + +The following statement sets the date and time after which the password is not valid: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl VALID UNTIL '2021-01-01'; +~~~ + +### Prevent a user from using password authentication + +The following statement prevents the user from using password authentication and mandates certificate-based client authentication: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl WITH PASSWORD NULL; +~~~ + +### Change login privileges for a user + +The following statement prevents the user from logging in with any [client authentication method](authentication.html#client-authentication): + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl NOLOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW USERS; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | NOLOGIN | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +The following statement allows the user to log in with one of the client authentication methods: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl LOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW USERS; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +## See also + +- [`DROP USER`](drop-user.html) +- [`SHOW USERS`](show-users.html) +- [`GRANT `](grant.html) +- [`SHOW GRANTS`](show-grants.html) +- [Create Security Certificates](cockroach-cert.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/alter-view.md b/v20.2/alter-view.md new file mode 100644 index 00000000000..48fee782202 --- /dev/null +++ b/v20.2/alter-view.md @@ -0,0 +1,81 @@ +--- +title: ALTER VIEW +summary: The ALTER VIEW statement changes the name of a view. +toc: true +--- + +The `ALTER VIEW` [statement](sql-statements.html) changes the name of a [view](views.html). + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +{{site.data.alerts.callout_info}} +It is not currently possible to change the `SELECT` statement executed by a view. Instead, you must drop the existing view and create a new view. Also, it is not currently possible to rename a view that other views depend on, but this ability may be added in the future (see [this issue](https://github.com/cockroachdb/cockroach/issues/10083)). +{{site.data.alerts.end}} + +## Required privileges + +The user must have the `DROP` [privilege](authorization.html#assign-privileges) on the view and the `CREATE` privilege on the parent database. + +## Synopsis + +
      + {% include {{ page.version.version }}/sql/diagrams/alter_view.html %} +
      + +## Parameters + +Parameter | Description +----------|------------ +`IF EXISTS` | Rename the view only if a view of `view_name` exists; if one does not exist, do not return an error. +`view_name` | The name of the view to rename. To find view names, use:

      `SELECT * FROM information_schema.tables WHERE table_type = 'VIEW';` +`name` | The new [`name`](sql-grammar.html#name) for the view, which must be unique to its database and follow these [identifier rules](keywords-and-identifiers.html#identifiers). + +## Example + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+--------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+--------------------+------------+---------+ +| def | bank | user_accounts | VIEW | 2 | +| def | bank | user_emails | VIEW | 1 | ++---------------+-------------------+--------------------+------------+---------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER VIEW bank.user_emails RENAME TO bank.user_email_addresses; +~~~ + +{% include copy-clipboard.html %} +~~~ +> RENAME VIEW +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+----------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+----------------------+------------+---------+ +| def | bank | user_accounts | VIEW | 2 | +| def | bank | user_email_addresses | VIEW | 3 | ++---------------+-------------------+----------------------+------------+---------+ +(2 rows) +~~~ + +## See also + +- [Views](views.html) +- [`CREATE VIEW`](create-view.html) +- [`SHOW CREATE`](show-create.html) +- [`DROP VIEW`](drop-view.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/architecture/distribution-layer.md b/v20.2/architecture/distribution-layer.md new file mode 100644 index 00000000000..a57293eaf84 --- /dev/null +++ b/v20.2/architecture/distribution-layer.md @@ -0,0 +1,186 @@ +--- +title: Distribution Layer +summary: The distribution layer of CockroachDB's architecture provides a unified view of your cluster's data. +toc: true +--- + +The distribution layer of CockroachDB's architecture provides a unified view of your cluster's data. + +{{site.data.alerts.callout_info}} +If you haven't already, we recommend reading the [Architecture Overview](overview.html). +{{site.data.alerts.end}} + +## Overview + +To make all data in your cluster accessible from any node, CockroachDB stores data in a monolithic sorted map of key-value pairs. This key-space describes all of the data in your cluster, as well as its location, and is divided into what we call "ranges", contiguous chunks of the key-space, so that every key can always be found in a single range. + +CockroachDB implements a sorted map to enable: + + - **Simple lookups**: Because we identify which nodes are responsible for certain portions of the data, queries are able to quickly locate where to find the data they want. + - **Efficient scans**: By defining the order of data, it's easy to find data within a particular range during a scan. + +### Monolithic sorted map structure + +The monolithic sorted map is comprised of two fundamental elements: + +- System data, which include **meta ranges** that describe the locations of data in your cluster (among many other cluster-wide and local data elements) +- User data, which store your cluster's **table data** + +#### Meta ranges + +The locations of all ranges in your cluster are stored in a two-level index at the beginning of your key-space, known as meta ranges, where the first level (`meta1`) addresses the second, and the second (`meta2`) addresses data in the cluster. Importantly, every node has information on where to locate the `meta1` range (known as its range descriptor, detailed below), and the range is never split. + +This meta range structure lets us address up to 4EiB of user data by default: we can address 2^(18 + 18) = 2^36 ranges; each range addresses 2^26 B, and altogether we address 2^(36+26) B = 2^62 B = 4EiB. However, with larger range sizes, it's possible to expand this capacity even further. + +Meta ranges are treated mostly like normal ranges and are accessed and replicated just like other elements of your cluster's KV data. + +Each node caches values of the `meta2` range it has accessed before, which optimizes access of that data in the future. Whenever a node discovers that its `meta2` cache is invalid for a specific key, the cache is updated by performing a regular read on the `meta2` range. + +#### Table data + +After the node's meta ranges is the KV data your cluster stores. + +Each table and its secondary indexes initially map to a single range, where each key-value pair in the range represents a single row in the table (also called the primary index because the table is sorted by the primary key) or a single row in a secondary index. As soon as a range reaches 512 MiB in size, it splits into two ranges. This process continues as a table and its indexes continue growing. Once a table is split across multiple ranges, it's likely that the table and secondary indexes will be stored in separate ranges. However, a range can still contain data for both the table and a secondary index. + +The default 512 MiB range size represents a sweet spot for us between a size that's small enough to move quickly between nodes, but large enough to store a meaningfully contiguous set of data whose keys are more likely to be accessed together. These ranges are then shuffled around your cluster to ensure survivability. + +These table ranges are replicated (in the aptly named replication layer), and have the addresses of each replica stored in the `meta2` range. + +### Using the monolithic sorted map + +When a node receives a request, it looks at the meta ranges to find out which node it needs to route the request to by comparing the keys in the request to the keys in its `meta2` range. + +These meta ranges are heavily cached, so this is normally handled without having to send an RPC to the node actually containing the `meta2` ranges. + +The node then sends those KV operations to the leaseholder identified in the `meta2` range. However, it's possible that the data moved, in which case the node that no longer has the information replies to the requesting node where it's now located. In this case we go back to the `meta2` range to get more up-to-date information and try again. + +### Interactions with other layers + +In relationship to other layers in CockroachDB, the distribution layer: + +- Receives requests from the transaction layer on the same node. +- Identifies which nodes should receive the request, and then sends the request to the proper node's replication layer. + +## Technical details and components + +### gRPC + +gRPC is the software nodes use to communicate with one another. Because the distribution layer is the first layer to communicate with other nodes, CockroachDB implements gRPC here. + +gRPC requires inputs and outputs to be formatted as protocol buffers (protobufs). To leverage gRPC, CockroachDB implements a protocol-buffer-based API defined in `api.proto`. + +For more information about gRPC, see the [official gRPC documentation](http://www.grpc.io/docs/guides/). + +### BatchRequest + +All KV operation requests are bundled into a [protobuf](https://en.wikipedia.org/wiki/Protocol_Buffers), known as a `BatchRequest`. The destination of this batch is identified in the `BatchRequest` header, as well as a pointer to the request's transaction record. (On the other side, when a node is replying to a `BatchRequest`, it uses a protobuf––`BatchResponse`.) + +This `BatchRequest` is also what's used to send requests between nodes using gRPC, which accepts and sends protocol buffers. + +### DistSender + +The gateway/coordinating node's `DistSender` receives `BatchRequest`s from its own `TxnCoordSender`. `DistSender` is then responsible for breaking up `BatchRequests` and routing a new set of `BatchRequests` to the nodes it identifies contain the data using its `meta2` ranges. It will use the cache to send the request to the leaseholder, but it's also prepared to try the other replicas, in order of "proximity." The replica that the cache says is the leaseholder is simply moved to the front of the list of replicas to be tried and then an RPC is sent to all of them, in order. + +Requests received by a non-leaseholder fail with an error pointing at the replica's last known leaseholder. These requests are retried transparently with the updated lease by the gateway node and never reach the client. + +As nodes begin replying to these commands, `DistSender` also aggregates the results in preparation for returning them to the client. + +### Meta range KV structure + +Like all other data in your cluster, meta ranges are structured as KV pairs. Both meta ranges have a similar structure: + +~~~ +metaX/successorKey -> LeaseholderAddress, [list of other nodes containing data] +~~~ + +Element | Description +--------|------------------------ +`metaX` | The level of meta range. Here we use a simplified `meta1` or `meta2`, but these are actually represented in `cockroach` as `\x02` and `\x03` respectively. +`successorKey` | The first key *greater* than the key you're scanning for. This makes CockroachDB's scans efficient; it simply scans the keys until it finds a value greater than the key it's looking for, and that is where it finds the relevant data.

      The `successorKey` for the end of a keyspace is identified as `maxKey`. +`LeaseholderAddress` | The replica primarily responsible for reads and writes, known as the leaseholder. The replication layer contains more information about [leases](replication-layer.html#leases). + +Here's an example: + +~~~ +meta2/M -> node1:26257, node2:26257, node3:26257 +~~~ + +In this case, the replica on `node1` is the leaseholder, and nodes 2 and 3 also contain replicas. + +#### Example + +Let's imagine we have an alphabetically sorted column, which we use for lookups. Here are what the meta ranges would approximately look like: + +1. `meta1` contains the address for the nodes containing the `meta2` replicas. + + ~~~ + # Points to meta2 range for keys [A-M) + meta1/M -> node1:26257, node2:26257, node3:26257 + + # Points to meta2 range for keys [M-Z] + meta1/maxKey -> node4:26257, node5:26257, node6:26257 + ~~~ + +2. `meta2` contains addresses for the nodes containing the replicas of each range in the cluster, the first of which is the [leaseholder](replication-layer.html#leases). + + ~~~ + # Contains [A-G) + meta2/G -> node1:26257, node2:26257, node3:26257 + + # Contains [G-M) + meta2/M -> node1:26257, node2:26257, node3:26257 + + #Contains [M-Z) + meta2/Z -> node4:26257, node5:26257, node6:26257 + + #Contains [Z-maxKey) + meta2/maxKey-> node4:26257, node5:26257, node6:26257 + ~~~ + +### Table data KV structure + +Key-value data, which represents the data in your tables using the following structure: + +~~~ +/
// -> +~~~ + +The table itself is stored with an `index_id` of 1 for its `PRIMARY KEY` columns, with the rest of the columns in the table considered as stored/covered columns. + +### Range descriptors + +Each range in CockroachDB contains metadata, known as a range descriptor. A range descriptor is comprised of the following: + +- A sequential RangeID +- The keyspace (i.e., the set of keys) the range contains; for example, the first and last `` in the table data KV structure above. This determines the `meta2` range's keys. +- The addresses of nodes containing replicas of the range, with its leaseholder (which is responsible for its reads and writes) in the first position. This determines the `meta2` range's key's values. + +Because range descriptors comprise the key-value data of the `meta2` range, each node's `meta2` cache also stores range descriptors. + +Range descriptors are updated whenever there are: + +- Membership changes to a range's Raft group (discussed in more detail in the [Replication Layer](replication-layer.html#membership-changes-rebalance-repair)) +- Leaseholder changes +- Range splits + +All of these updates to the range descriptor occur locally on the range, and then propagate to the `meta2` range. + +### Range splits + +By default, CockroachDB attempts to keep ranges/replicas at 512 MiB. Once a range reaches that limit we split it into two smaller ranges (composed of contiguous key spaces). + +During this range split, the node creates a new Raft group containing all of the same members as the range that was split. The fact that there are now two ranges also means that there is a transaction that updates `meta2` with the new keyspace boundaries, as well as the addresses of the nodes using the range descriptor. + +## Technical interactions with other layers + +### Distribution and transaction layer + +The Distribution layer's `DistSender` receives `BatchRequests` from its own node's `TxnCoordSender`, housed in the Transaction layer. + +### Distribution and replication layer + +The Distribution layer routes `BatchRequests` to nodes containing ranges of data, which is ultimately routed to the Raft group leader or leaseholder, which are handled in the replication layer. + +## What's next? + +Learn how CockroachDB copies data and ensures consistency in the [replication layer](replication-layer.html). diff --git a/v20.2/architecture/life-of-a-distributed-transaction.md b/v20.2/architecture/life-of-a-distributed-transaction.md new file mode 100644 index 00000000000..1931c3e9ad6 --- /dev/null +++ b/v20.2/architecture/life-of-a-distributed-transaction.md @@ -0,0 +1,189 @@ +--- +title: Life of a Distributed Transaction +summary: This guide details the path through CockroachDB's architecture that a query takes, starting with a SQL client and progressing all the way to RocksDB (and then back out again). +toc: true +--- + +Because CockroachDB is a distributed transactional database, the path queries take is dramatically different from many other database architectures. To help familiarize you with CockroachDB's internals, this guide covers what that path actually is. + +If you've already read the [CockroachDB architecture documentation](overview.html), this guide serves as another way to conceptualize how the database works. This time, instead of focusing on the layers of CockroachDB's architecture, we're going to focus on the linear path that a query takes through the system (and then back out again). + +To get the most out of this guide, we recommend beginning with the architecture documentation's [overview](overview.html) and progressing through all of the following sections. This guide provides brief descriptions of each component's function and links to other documentation where appropriate, but assumes the reader has a basic understanding of the architecture in the first place. + +## Overview + +This guide is organized by the physical actors in the system, and then broken down into the components of each actor in the sequence in which they're involved. + +Here's a brief overview of the physical actors, in the sequence with which they're involved in executing a query: + +1. [**SQL Client**](#sql-client-postgres-wire-protocol) sends a query to your cluster. +1. [**Load Balancing**](#load-balancing-routing) routes the request to CockroachDB nodes in your cluster, which will act as a gateway. +1. [**Gateway**](#gateway) is a CockroachDB node that processes the SQL request and responds to the client. +1. [**Leaseholder**](#leaseholder-node) is a CockroachDB node responsible for serving reads and coordinating writes of a specific range of keys in your query. +1. [**Raft leader**](#raft-leader) is a CockroachDB node responsible for maintaining consensus among your CockroachDB replicas. + +Once the transaction completes, queries traverse these actors in approximately reverse order. We say "approximately" because there might be many leaseholders and Raft leaders involved in a single query, and there is little-to-no interaction with the load balancer during the response. + +## SQL Client/Postgres Wire Protocol + +To begin, a SQL client (e.g., an app) performs some kind of business logic against your CockroachDB cluster, such as inserting a new customer record. + +This request is sent over a connection to your CockroachDB cluster that's established using a PostgreSQL driver. + +## Load Balancing & Routing + +Modern architectures require distributing your cluster across machines to improve throughput, latency, and uptime. This means queries are routed through load balancers, which choose the best CockroachDB node to connect to. Because all CockroachDB nodes have perfectly symmetrical access to data, this means your load balancer can connect your client to any node in the cluster and access any data while still guaranteeing strong consistency. + +Your architecture might also have additional layers of routing to enforce regulatory compliance, such as ensuring GDPR compliance. + +Once your router and load balancer determine the best node to connect to, your client's connection is established to the gateway node. + +## Gateway + +The gateway node handles the connection with the client, both receiving and responding to the request. + +### SQL parsing & planning + +The gateway node first [parses](sql-layer.html#sql-parser-planner-executor) the client's SQL statement to ensure it's valid according to the CockroachDB dialect of SQL, and uses that information to [generate a logical SQL plan](sql-layer.html#logical-planning). + +Given that CockroachDB is a distributed database, though, it's also important to take a cluster's topology into account, so the logical plan is then converted into a physical plan—this means sometimes pushing operations onto the physical machines that contain the data. + +### SQL executor + +While CockroachDB presents a SQL interface to clients, the actual database is built on top of a key-value store. To mediate this, the physical plan generated at the end of SQL parsing is passed to the SQL executor, which executes the plan by performing key-value operations through the `TxnCoordSender`. For example, the SQL executor converts `INSERT `statements into `Put()` operations. + +### TxnCoordSender + +The `TxnCoordSender` provides an API to perform key-value operations on your database. + +On its back end, the `TxnCoordSender` performs a large amount of the accounting and tracking for a transaction, including: + +- Accounts for all keys involved in a transaction. This is used, among other ways, to manage the transaction's state. +- Packages all key-value operations into a `BatchRequest`, which are forwarded on to the node's `DistSender`. + +### DistSender + +The gateway node's `DistSender` receives `BatchRequests` from the `TxnCoordSender`. It dismantles the initial `BatchRequest` by taking each operation and finding which physical machine should receive the request for the range—known as the range's leaseholder. The address of the range's current leaseholder is readily available in both local caches, as well as in the [cluster's `meta` ranges](distribution-layer.html#meta-range-kv-structure). + +These dismantled `BatchRequests` are reassembled into new `BatchRequests` containing the address of the range's leaseholder. + +All write operations also propagate the leaseholder's address back to the `TxnCoordSender`, so it can track and clean up write operations as necessary. + +The `DistSender` sends out the first `BatchRequest` for each range in parallel. As soon as it receives a provisional acknowledgment from the leaseholder node’s evaluator (details below), it sends out the next `BatchRequest` for that range. + +The `DistSender` then waits to receive acknowledgments for all of its write operations, as well as values for all of its read operations. However, this wait isn't necessarily blocking, and the `DistSender` may still perform operations with ongoing transactions. + +## Leaseholder node + +The gateway node's `DistSender` tries to send its `BatchRequests` to the replica identified as the range's [leaseholder](replication-layer.html#leases), which is a single replica that serves all reads for a range, as well as coordinates all writes. Leaseholders play a crucial role in CockroachDB's architecture, so it's a good topic to make sure you're familiar with. + +### Request response + +Because the leaseholder replica can shift between nodes, all nodes must be able to return a request for any key, returning a response indicating one of these scenarios: + +##### No Longer Leaseholder + +If a node is no longer the leaseholder, but still contains a replica of the range, it denies the request but includes the last known address for the leaseholder of that range. + +Upon receipt of this response, the `DistSender` will update the header of the `BatchRequest` with the new address, and then resend the `BatchRequest` to the newly identified leaseholder. + +##### No Longer Has/Never Had Range + +If a node doesn't have a replica for the requested range, it denies the request without providing any further information. + +In this case, the `DistSender` must look up the current leaseholder using the [cluster's `meta` ranges](distribution-layer.html#meta-range-kv-structure). + +##### Success + +Once the node that contains the leaseholder of the range receives the `BatchRequest`, it begins processing it, and progresses onto checking the timestamp cache. + +### Timestamp cache + +The timestamp cache tracks the highest timestamp (i.e., most recent) for any read operation that a given range has served. + +Each write operation in a `BatchRequest` checks its own timestamp versus the timestamp cache to ensure that the write operation has a higher timestamp; this guarantees that history is never rewritten and you can trust that reads always served the most recent data. It's one of the crucial mechanisms CockroachDB uses to ensure serializability. If a write operation fails this check, it must be restarted at a timestamp higher than the timestamp cache's value. + +### Latch manager + +Operations in the `BatchRequest` are serialized through the leaseholder's latch manager. + +This works by giving each write operation a latch on a row. Any reads or writes that come in after the latch has been granted on the row must wait for the write to complete, at which point the latch is released and the subsequent operations can continue. + +### Batch Evaluation + +The batch evaluator ensures that write operations are valid. Our architecture makes this fairly trivial. First, the evaluator can simply check the leaseholder's data to ensure the write is valid; because it has coordinated all writes to the range, it must have the most up-to-date versions of the range's data. Secondly, because of the latch manager, each write operation is guaranteed to uncontested access to the range (i.e., there is no contention with other write operations). + +If the write operation is valid according to the evaluator, the leaseholder sends a provisional acknowledgment to the gateway node's `DistSender`; this lets the `DistSender` begin to send its subsequent `BatchRequests` for this range. + +Importantly, this feature is entirely built for transactional optimization (known as [transaction pipelining](transaction-layer.html#transaction-pipelining)). There are no issues if an operation passes the evaluator but doesn't end up committing. + +### Reads from RocksDB + +All operations (including writes) begin by reading from the local instance of RocksDB to check for write intents for the operation's key. We talk much more about [write intents in the transaction layer of the CockroachDB architecture](transaction-layer.html#write-intents), which is worth reading, but a simplified explanation is that these are provisional, uncommitted writes that express that some other concurrent transaction plans to write a value to the key. + +What we detail below is a simplified version of the CockroachDB transaction model. For more detail, check out [the transaction architecture documentation](transaction-layer.html). + +#### Resolving Write Intents + +If an operation encounters a write intent for a key, it attempts to "resolve" the write intent by checking the state of the write intent's transaction. If the write's intent's transaction record is... + +- `COMMITTED`, this operation converts the write intent to a regular key-value pair, and then proceeds as if it had read that value instead of a write intent. +- `ABORTED`, this operation discards the write intent and reads the next-most-recent value from RocksDB. +- `PENDING`, the new transaction attempts to "push" the write intent's transaction by moving that transaction's timestamp forward (i.e., ahead of this transaction's timestamp); however, this only succeeds if the write intent's transaction has become inactive. + + If the push succeeds, the operation continues. + + If this push fails (which is the majority of the time), this transaction goes into the [`TxnWaitQueue`](https://www.cockroachlabs.com/docs/stable/architecture/transaction-layer.html#txnwaitqueue) on this node. The incoming transaction can only continue once the blocking transaction completes (i.e., commits or aborts). +- _Missing_, the resolver consults the write intent's timestamp. If it was created within the transaction liveness threshold, it treats the transaction record as exhibiting the `PENDING` behavior, with the addition of tracking the push in the range's timestamp cache, which will inform the transaction that its timestamp was pushed once the transaction record gets created. + + If the write intent is older than the transaction liveness threshold, the resolution exhibits the `ABORTED` behavior. + + Note that transaction records might be missing because we've avoided writing the record until the transaction commits. For more information, see [Transaction Layer: Transaction records](transaction-layer.html#transaction-records). + +Check out our architecture documentation for more information about [CockroachDB's transactional model](transaction-layer.html). + +#### Read Operations + +If the read doesn't encounter a write intent and the key-value operation is meant to serve a read, it can simply use the value it read from the leaseholder's instance of RocksDB. This works because the leaseholder had to be part of the Raft consensus group for any writes to complete, meaning it must have the most up-to-date version of the range's data. + +The leaseholder aggregates all read responses into a `BatchResponse` that will get returned to the gateway node's `DistSender`. + +As we mentioned before, each read operation also updates the timestamp cache. + +### Write Operations + +After guaranteeing that there are no existing write intents for the keys, `BatchRequest`'s key-value operations are converted to [Raft operations](replication-layer.html#raft) and have their values converted into write intents. + +The leaseholder then proposes these Raft operations to the Raft group leader. The leaseholder and the Raft leader are almost always the same node, but there are situations where the roles might drift to different nodes. However, when the two roles are not collocated on the same physical machine, CockroachDB will attempt to relocate them on the same node at the next opportunity. + +## Raft Leader + +CockroachDB leverages Raft as its consensus protocol. If you aren't familiar with it, we recommend checking out the details about [how CockroachDB leverages Raft](https://www.cockroachlabs.com/docs/v2.1/architecture/replication-layer.html#raft), as well as [learning more about how the protocol works at large](http://thesecretlivesofdata.com/raft/). + +In terms of executing transactions, the Raft leader receives proposed Raft commands from the leaseholder. Each Raft command is a write that is used to represent an atomic state change of the underlying key-value pairs stored in RocksDB. + +### Consensus + +For each command the Raft leader receives, it proposes a vote to the other members of the Raft group. + +Once the command achieves consensus (i.e., a majority of nodes including itself acknowledge the Raft command), it is committed to the Raft leader’s Raft log and written to RocksDB. At the same time, the Raft leader also sends a command to all other nodes to include the command in their Raft logs. + +Once the leader commits the Raft log entry, it’s considered committed. At this point the value is considered written, and if another operation comes in and performs a read on RocksDB for this key, they’ll encounter this value. + +Note that this write operation creates a write intent; these writes will not be fully committed until the gateway node sets the transaction record's status to `COMMITTED`. + +## On the way back up + +Now that we have followed an operation all the way down from the SQL client to RocksDB, we can pretty quickly cover what happens on the way back up (i.e., when generating a response to the client). + +1. Once the leaseholder applies a write to its Raft log, + it sends an commit acknowledgment to the gateway node's `DistSender`, which was waiting for this signal (having already received the provisional acknowledgment from the leaseholder's evaluator). +1. The gateway node's `DistSender` aggregates commit acknowledgments from all of the write operations in the `BatchRequest`, as well as any values from read operations that should be returned to the client. +1. Once all operations have successfully completed (i.e., reads have returned values and write intents have been committed), the `DistSender` tries to record the transaction's success in the transaction record (which provides a durable mechanism of tracking the transaction's state), which can cause a few situations to arise: + - It checks the timestamp cache of the range where the first write occurred to see if its timestamp got pushed forward. If it did, the transaction performs a [read refresh](transaction-layer.html#read-refreshing) to see if any values it needed have been changed. If the read refresh is successful, the transaction can commit at the pushed timestamp. If the read refresh fails, the transaction must be restarted. + - If the transaction is in an `ABORTED` state, the `DistSender` sends a response indicating as much, which ends up back at the SQL interface. + + Upon passing these checks the transaction record is either written for the first time with the `COMMITTED` state, or if it was in a `PENDING` state, it is moved to `COMMITTED`. Only at this point is the transaction considered committed. +1. The `DistSender` propagates any values that should be returned to the client (e.g., reads or the number of affected rows) to the `TxnCoordSender`, which in turn responds to the SQL interface with the value. + The `TxnCoordSender` also begins asynchronous intent cleanup by sending a request to the `DistSender` to convert all write intents it created for the transaction to fully committed values. However, this process is largely an optimization; if any operation encounters a write intent, it checks the write intent's transaction record. If the transaction record is `COMMITTED`, the operation can perform the same cleanup and convert the write intent to a fully committed value. +1. The SQL interface then responds to the client, and is now prepared to continue accepting new connections. diff --git a/v20.2/architecture/overview.md b/v20.2/architecture/overview.md new file mode 100644 index 00000000000..b093fc868e9 --- /dev/null +++ b/v20.2/architecture/overview.md @@ -0,0 +1,88 @@ +--- +title: Architecture Overview +summary: An overview of the CockroachDB architecture. +toc: true +key: cockroachdb-architecture.html +redirect_from: index.html +--- + +CockroachDB was designed to create the source-available database our developers would want to use: one that is both scalable and consistent. Developers often have questions about how we've achieved this, and this guide sets out to detail the inner-workings of the `cockroach` process as a means of explanation. + +However, you definitely do not need to understand the underlying architecture to use CockroachDB. These pages give serious users and database enthusiasts a high-level framework to explain what's happening under the hood. + +## Using this guide + +This guide is broken out into pages detailing each layer of CockroachDB. It's recommended to read through the layers sequentially, starting with this overview and then proceeding to the SQL layer. + +If you're looking for a high-level understanding of CockroachDB, you can simply read the **Overview** section of each layer. For more technical detail––for example, if you're interested in [contributing to the project](https://wiki.crdb.io/wiki/spaces/CRDB/pages/73204033/Contributing+to+CockroachDB)––you should read the **Components** sections as well. + +{{site.data.alerts.callout_info}}This guide details how CockroachDB is built, but does not explain how you should architect an application using CockroachDB. For help with your own application's architecture using CockroachDB, check out our user documentation.{{site.data.alerts.end}} + +## Goals of CockroachDB + +CockroachDB was designed in service of the following goals: + +- Make life easier for humans. This means being low-touch and highly automated for operators and simple to reason about for developers. +- Offer industry-leading consistency, even on massively scaled deployments. This means enabling distributed transactions, as well as removing the pain of eventual consistency issues and stale reads. +- Create an always-on database that accepts reads and writes on all nodes without generating conflicts. +- Allow flexible deployment in any environment, without tying you to any platform or vendor. +- Support familiar tools for working with relational data (i.e., SQL). + +With the confluence of these features, we hope that CockroachDB lets teams easily build global, scalable, resilient cloud services. + +## Glossary + +### Terms + +It's helpful to understand a few terms before reading our architecture documentation. + +{% include {{ page.version.version }}/misc/basic-terms.md %} + +### Concepts + +CockroachDB heavily relies on the following concepts, so being familiar with them will help you understand what our architecture achieves. + +Term | Definition +-----|----------- +**Consistency** | CockroachDB uses "consistency" in both the sense of [ACID semantics](https://en.wikipedia.org/wiki/Consistency_(database_systems)) and the [CAP theorem](https://en.wikipedia.org/wiki/CAP_theorem), albeit less formally than either definition. What we try to express with this term is that your data should be anomaly-free. +**Consensus** | When a range receives a write, a quorum of nodes containing replicas of the range acknowledge the write. This means your data is safely stored and a majority of nodes agree on the database's current state, even if some of the nodes are offline.

When a write *doesn't* achieve consensus, forward progress halts to maintain consistency within the cluster. +**Replication** | Replication involves creating and distributing copies of data, as well as ensuring copies remain consistent. However, there are multiple types of replication: namely, synchronous and asynchronous.

Synchronous replication requires all writes to propagate to a quorum of copies of the data before being considered committed. To ensure consistency with your data, this is the kind of replication CockroachDB uses.

Asynchronous replication only requires a single node to receive the write to be considered committed; it's propagated to each copy of the data after the fact. This is more or less equivalent to "eventual consistency", which was popularized by NoSQL databases. This method of replication is likely to cause anomalies and loss of data. +**Transactions** | A set of operations performed on your database that satisfy the requirements of [ACID semantics](https://en.wikipedia.org/wiki/Database_transaction). This is a crucial component for a consistent system to ensure developers can trust the data in their database. +**Multi-Active Availability** | Our consensus-based notion of high availability that lets each node in the cluster handle reads and writes for a subset of the stored data (on a per-range basis). This is in contrast to active-passive replication, in which the active node receives 100% of request traffic, as well as active-active replication, in which all nodes accept requests but typically cannot guarantee that reads are both up-to-date and fast. + +## Overview + +CockroachDB starts running on machines with two commands: + +- `cockroach start` with a `--join` flag for all of the initial nodes in the cluster, so the process knows all of the other machines it can communicate with +- `cockroach init` to perform a one-time initialization of the cluster + +Once the `cockroach` process is running, developers interact with CockroachDB through a SQL API, which we've modeled after PostgreSQL. Thanks to the symmetrical behavior of all nodes, you can send SQL requests to any of them; this makes CockroachDB really easy to integrate with load balancers. + +After receiving SQL RPCs, nodes convert them into operations that work with our distributed key-value store. As these RPCs start filling your cluster with data, CockroachDB algorithmically starts distributing your data among your nodes, breaking the data up into 512 MiB chunks that we call ranges. Each range is replicated to at least 3 nodes to ensure survivability. This way, if nodes go down, you still have copies of the data which can be used for reads and writes, as well as replicating the data to other nodes. + +If a node receives a read or write request it cannot directly serve, it simply finds the node that can handle the request, and communicates with it. This way you do not need to know where your data lives, CockroachDB tracks it for you, and enables symmetric behavior for each node. + +Any changes made to the data in a range rely on a consensus algorithm to ensure a majority of its replicas agree to commit the change, ensuring industry-leading isolation guarantees and providing your application consistent reads, regardless of which node you communicate with. + +Ultimately, data is written to and read from disk using an efficient storage engine, which is able to keep track of the data's timestamp. This has the benefit of letting us support the SQL standard `AS OF SYSTEM TIME` clause, letting you find historical data for a period of time. + +However, while that high-level overview gives you a notion of what CockroachDB does, looking at how the `cockroach` process operates on each of these needs will give you much greater understanding of our architecture. + +### Layers + +At the highest level, CockroachDB converts clients' SQL statements into key-value (KV) data, which is distributed among nodes and written to disk. Our architecture is the process by which we accomplish that, which is manifested as a number of layers that interact with those directly above and below it as relatively opaque services. + +The following pages describe the function each layer performs, but mostly ignore the details of other layers. This description is true to the experience of the layers themselves, which generally treat the other layers as black-box APIs. There are interactions that occur between layers which *are not* clearly articulated and require an understanding of each layer's function to understand the entire process. + +Layer | Order | Purpose +------|------------|-------- +[SQL](sql-layer.html) | 1 | Translate client SQL queries to KV operations. +[Transactional](transaction-layer.html) | 2 | Allow atomic changes to multiple KV entries. +[Distribution](distribution-layer.html) | 3 | Present replicated KV ranges as a single entity. +[Replication](replication-layer.html) | 4 | Consistently and synchronously replicate KV ranges across many nodes. This layer also enables consistent reads via leases. +[Storage](storage-layer.html) | 5 | Write and read KV data on disk. + +## What's next? + +Begin understanding our architecture by learning how CockroachDB works with applications in the [SQL layer](sql-layer.html). diff --git a/v20.2/architecture/reads-and-writes-overview.md b/v20.2/architecture/reads-and-writes-overview.md new file mode 100644 index 00000000000..da5f959f6e2 --- /dev/null +++ b/v20.2/architecture/reads-and-writes-overview.md @@ -0,0 +1,67 @@ +--- +title: Reads and Writes in CockroachDB +summary: Learn how reads and writes are affected by the replicated and distributed nature of data in CockroachDB. +toc: true +--- + +This page explains how reads and writes are affected by the replicated and distributed nature of data in CockroachDB. It starts by summarizing some important [CockroachDB architectural concepts](overview.html) and then walks you through a few simple read and write scenarios. + +{{site.data.alerts.callout_info}} +For a more detailed trace of a query through the layers of CockroachDB's architecture, see [Life of a Distributed Transaction](life-of-a-distributed-transaction.html). +{{site.data.alerts.end}} + +## Important concepts + +{% include {{ page.version.version }}/misc/basic-terms.md %} + +As mentioned above, when a query is executed, the cluster routes the request to the leaseholder for the range containing the relevant data. If the query touches multiple ranges, the request goes to multiple leaseholders. For a read request, only the leaseholder of the relevant range retrieves the data. For a write request, the Raft consensus protocol dictates that a majority of the replicas of the relevant range must agree before the write is committed. + +Let's consider how these mechanics play out in some hypothetical queries. + +## Read scenario + +First, imagine a simple read scenario where: + +- There are 3 nodes in the cluster. +- There are 3 small tables, each fitting in a single range. +- Ranges are replicated 3 times (the default). +- A query is executed against node 2 to read from table 3. + +Perf tuning concepts + +In this case: + +1. Node 2 (the gateway node) receives the request to read from table 3. +2. The leaseholder for table 3 is on node 3, so the request is routed there. +3. Node 3 returns the data to node 2. +4. Node 2 responds to the client. + +If the query is received by the node that has the leaseholder for the relevant range, there are fewer network hops: + +Perf tuning concepts + +## Write scenario + +Now imagine a simple write scenario where a query is executed against node 3 to write to table 1: + +Perf tuning concepts + +In this case: + +1. Node 3 (the gateway node) receives the request to write to table 1. +2. The leaseholder for table 1 is on node 1, so the request is routed there. +3. The leaseholder is the same replica as the Raft leader (as is typical), so it simultaneously appends the write to its own Raft log and notifies its follower replicas on nodes 2 and 3. +4. As soon as one follower has appended the write to its Raft log (and thus a majority of replicas agree based on identical Raft logs), it notifies the leader and the write is committed to the key-values on the agreeing replicas. In this diagram, the follower on node 2 acknowledged the write, but it could just as well have been the follower on node 3. Also note that the follower not involved in the consensus agreement usually commits the write very soon after the others. +5. Node 1 returns acknowledgement of the commit to node 3. +6. Node 3 responds to the client. + +Just as in the read scenario, if the write request is received by the node that has the leaseholder and Raft leader for the relevant range, there are fewer network hops: + +Perf tuning concepts + +## Network and I/O bottlenecks + +With the above examples in mind, it's always important to consider network latency and disk I/O as potential performance bottlenecks. In summary: + +- For reads, hops between the gateway node and the leaseholder add latency. +- For writes, hops between the gateway node and the leaseholder/Raft leader, and hops between the leaseholder/Raft leader and Raft followers, add latency. In addition, since Raft log entries are persisted to disk before a write is committed, disk I/O is important. diff --git a/v20.2/architecture/replication-layer.md b/v20.2/architecture/replication-layer.md new file mode 100644 index 00000000000..e73944f6eed --- /dev/null +++ b/v20.2/architecture/replication-layer.md @@ -0,0 +1,154 @@ +--- +title: Replication Layer +summary: The replication layer of CockroachDB's architecture copies data between nodes and ensures consistency between copies. +toc: true +--- + +The replication layer of CockroachDB's architecture copies data between nodes and ensures consistency between these copies by implementing our consensus algorithm. + +{{site.data.alerts.callout_info}} +If you haven't already, we recommend reading the [Architecture Overview](overview.html). +{{site.data.alerts.end}} + +## Overview + +High availability requires that your database can tolerate nodes going offline without interrupting service to your application. This means replicating data between nodes to ensure the data remains accessible. + +Ensuring consistency with nodes offline, though, is a challenge many databases fail. To solve this problem, CockroachDB uses a consensus algorithm to require that a quorum of replicas agrees on any changes to a range before those changes are committed. Because 3 is the smallest number that can achieve quorum (i.e., 2 out of 3), CockroachDB's high availability (known as multi-active availability) requires 3 nodes. + +The number of failures that can be tolerated is equal to *(Replication factor - 1)/2*. For example, with 3x replication, one failure can be tolerated; with 5x replication, two failures, and so on. You can control the replication factor at the cluster, database, and table level using [replication zones](../configure-replication-zones.html). + +When failures happen, though, CockroachDB automatically realizes nodes have stopped responding and works to redistribute your data to continue maximizing survivability. This process also works the other way around: when new nodes join your cluster, data automatically rebalances onto it, ensuring your load is evenly distributed. + +### Interactions with other layers + +In relationship to other layers in CockroachDB, the replication layer: + +- Receives requests from and sends responses to the distribution layer. +- Writes accepted requests to the storage layer. + +## Components + +### Raft + +Raft is a consensus protocol––an algorithm which makes sure that your data is safely stored on multiple machines, and that those machines agree on the current state even if some of them are temporarily disconnected. + +Raft organizes all nodes that contain a replica of a range into a group--unsurprisingly called a Raft group. Each replica in a Raft group is either a "leader" or a "follower". The leader, which is elected by Raft and long-lived, coordinates all writes to the Raft group. It heartbeats followers periodically and keeps their logs replicated. In the absence of heartbeats, followers become candidates after randomized election timeouts and proceed to hold new leader elections. + +Once a node receives a `BatchRequest` for a range it contains, it converts those KV operations into Raft commands. Those commands are proposed to the Raft group leader––which is what makes it ideal for the [leaseholder](#leases) and the Raft leader to be one in the same––and written to the Raft log. + +For a great overview of Raft, we recommend [The Secret Lives of Data](http://thesecretlivesofdata.com/raft/). + +#### Raft logs + +When writes receive a quorum, and are committed by the Raft group leader, they're appended to the Raft log. This provides an ordered set of commands that the replicas agreed on and is essentially the source of truth for consistent replication. + +Because this log is treated as serializable, it can be replayed to bring a node from a past state to its current state. This log also lets nodes that temporarily went offline to be "caught up" to the current state without needing to receive a copy of the existing data in the form of a snapshot. + +### Snapshots + +Each replica can be "snapshotted", which copies all of its data as of a specific timestamp (available because of [MVCC](storage-layer.html#mvcc)). This snapshot can be sent to other nodes during a rebalance event to expedite replication. + +After loading the snapshot, the node gets up to date by replaying all actions from the Raft group's log that have occurred since the snapshot was taken. + +### Leases + +A single node in the Raft group acts as the leaseholder, which is the only node that can serve reads or propose writes to the Raft group leader (both actions are received as `BatchRequests` from [`DistSender`](distribution-layer.html#distsender)). + +When serving reads, leaseholders bypass Raft; for the leaseholder's writes to have been committed in the first place, they must have already achieved consensus, so a second consensus on the same data is unnecessary. This has the benefit of not incurring networking round trips required by Raft and greatly increases the speed of reads (without sacrificing consistency). + +CockroachDB attempts to elect a leaseholder who is also the Raft group leader, which can also optimize the speed of writes. + +If there is no leaseholder, any node receiving a request will attempt to become the leaseholder for the range. To prevent two nodes from acquiring the lease, the requester includes a copy of the last valid lease it had; if another node became the leaseholder, its request is ignored. + +#### Co-location with Raft leadership + +The range lease is completely separate from Raft leadership, and so without further efforts, Raft leadership and the Range lease might not be held by the same replica. However, we can optimize query performance by making the same node both Raft leader and the leaseholder; it reduces network round trips if the leaseholder receiving the requests can simply propose the Raft commands to itself, rather than communicating them to another node. + +To achieve this, each lease renewal or transfer also attempts to collocate them. In practice, that means that the mismatch is rare and self-corrects quickly. + +#### Epoch-based leases (table data) + +To manage leases for table data, CockroachDB implements a notion of "epochs," which are defined as the period between a node joining a cluster and a node disconnecting from a cluster. To extend its leases, each node must periodically update its liveness record, which is stored on a system range key. When a node disconnects, it stops updating the liveness record, and the the epoch is considered changed. This causes the node to immediately lose all of its leases. + +Because leases don't expire until a node disconnects from a cluster, leaseholders do not have to individually renew their own leases. Tying lease lifetimes to node liveness in this way lets us eliminate a substantial amount of traffic and Raft processing we would otherwise incur, while still tracking leases for every range. + +#### Expiration-based leases (meta and system ranges) + +A table's meta and system ranges (detailed in the [distribution layer](distribution-layer.html#meta-ranges)) are treated as normal key-value data, and therefore have leases just like table data. + +However, unlike table data, system ranges cannot use epoch-based leases because that would create a circular dependency: system ranges are already being used to implement epoch-based leases for table data. Therefore, system ranges use expiration-based leases instead. Expiration-based leases expire at a particular timestamp (typically after a few seconds). However, as long as a node continues proposing Raft commands, it continues to extend the expiration of its leases. If it doesn't, the next node containing a replica of the range that tries to read from or write to the range will become the leaseholder. + +#### Leaseholder rebalancing + +Because CockroachDB serves reads from a range's leaseholder, it benefits your cluster's performance if the replica closest to the primary geographic source of traffic holds the lease. However, as traffic to your cluster shifts throughout the course of the day, you might want to dynamically shift which nodes hold leases. + +{{site.data.alerts.callout_info}} + +This feature is also called [Follow-the-Workload](../demo-follow-the-workload.html) in our documentation. + +{{site.data.alerts.end}} + +Periodically (every 10 minutes by default in large clusters, but more frequently in small clusters), each leaseholder considers whether it should transfer the lease to another replica by considering the following inputs: + +- Number of requests from each locality +- Number of leases on each node +- Latency between localities + +##### Intra-locality + +If all the replicas are in the same locality, the decision is made entirely on the basis of the number of leases on each node that contains a replica, trying to achieve a roughly equitable distribution of leases across all of them. This means the distribution isn't perfectly equal; it intentionally tolerates small deviations between nodes to prevent thrashing (i.e., excessive adjustments trying to reach an equilibrium). + +##### Inter-locality + +If replicas are in different localities, CockroachDB attempts to calculate which replica would make the best leaseholder, i.e., provide the lowest latency. + +To enable dynamic leaseholder rebalancing, a range's current leaseholder tracks how many requests it receives from each locality as an exponentially weighted moving average. This calculation results in the locality that has recently requested the range most often being assigned the greatest weight. If another locality then begins requesting the range very frequently, this calculation would shift to assign the second region the greatest weight. + +When checking for leaseholder rebalancing opportunities, the leaseholder correlates each requesting locality's weight (i.e., the proportion of recent requests) to the locality of each replica by checking how similar the localities are. For example, if the leaseholder received requests from gateway nodes in locality `country=us,region=central`, CockroachDB would assign the following weights to replicas in the following localities: + +Replica locality | Replica rebalancing weight +-----------------|------------------- +`country=us,region=central` | 100% because it is an exact match +`country=us,region=east` | 50% because only the first locality matches +`country=aus,region=central` | 0% because the first locality does not match + +The leaseholder then evaluates its own weight and latency versus the other replicas to determine an adjustment factor. The greater the disparity between weights and the larger the latency between localities, the more CockroachDB favors the node from the locality with the larger weight. + +When checking for leaseholder rebalancing opportunities, the current leaseholder evaluates each replica's rebalancing weight and adjustment factor for the localities with the greatest weights. If moving the leaseholder is both beneficial and viable, the current leaseholder will transfer the lease to the best replica. + +##### Controlling leaseholder rebalancing + +You can control leaseholder rebalancing through the `kv.allocator.load_based_lease_rebalancing.enabled` and `kv.allocator.lease_rebalancing_aggressiveness` [cluster settings](../cluster-settings.html). Note that depending on the needs of your deployment, you can exercise additional control over the location of leases and replicas by [configuring replication zones](../configure-replication-zones.html). + +### Membership changes: rebalance/repair + +Whenever there are changes to a cluster's number of nodes, the members of Raft groups change and, to ensure optimal survivability and performance, replicas need to be rebalanced. What that looks like varies depending on whether the membership change is nodes being added or going offline. + +- **Nodes added**: The new node communicates information about itself to other nodes, indicating that it has space available. The cluster then rebalances some replicas onto the new node. + +- **Nodes going offline**: If a member of a Raft group ceases to respond, after 5 minutes, the cluster begins to rebalance by replicating the data the downed node held onto other nodes. + +Rebalancing is achieved by using a snapshot of a replica from the leaseholder, and then sending the data to another node over [gRPC](distribution-layer.html#grpc). After the transfer has been completed, the node with the new replica joins that range's Raft group; it then detects that its latest timestamp is behind the most recent entries in the Raft log and it replays all of the actions in the Raft log on itself. + +#### Load-based replica rebalancing + +In addition to the rebalancing that occurs when nodes join or leave a cluster, replicas are also rebalanced automatically based on the relative load across the nodes within a cluster. For more information, see the `kv.allocator.load_based_rebalancing` and `kv.allocator.qps_rebalance_threshold` [cluster settings](../cluster-settings.html). Note that depending on the needs of your deployment, you can exercise additional control over the location of leases and replicas by [configuring replication zones](../configure-replication-zones.html). + +## Interactions with other layers + +### Replication and distribution layers + +The replication layer receives requests from its and other nodes' `DistSender`. If this node is the leaseholder for the range, it accepts the requests; if it isn't, it returns an error with a pointer to which node it believes *is* the leaseholder. These KV requests are then turned into Raft commands. + +The replication layer sends `BatchResponses` back to the distribution layer's `DistSender`. + +### Replication and storage layers + +Committed Raft commands are written to the Raft log and ultimately stored on disk through the storage layer. + +The leaseholder serves reads from its RocksDB instance, which is in the storage layer. + +## What's next? + +Learn how CockroachDB reads and writes data from disk in the [storage layer](storage-layer.html). diff --git a/v20.2/architecture/sql-layer.md b/v20.2/architecture/sql-layer.md new file mode 100644 index 00000000000..21c3c79d3f3 --- /dev/null +++ b/v20.2/architecture/sql-layer.md @@ -0,0 +1,126 @@ +--- +title: SQL Layer +summary: The SQL layer of CockroachDB's architecture exposes its SQL API to developers and converts SQL statements into key-value operations. +toc: true +--- + +The SQL layer of CockroachDB's architecture exposes its SQL API to developers and converts SQL statements into key-value operations used by the rest of the database. + +{{site.data.alerts.callout_info}} +If you haven't already, we recommend reading the [Architecture Overview](overview.html). +{{site.data.alerts.end}} + +## Overview + +Once CockroachDB has been deployed, developers need nothing more than a connection string to the cluster and SQL statements to start working. + +Because CockroachDB's nodes all behave symmetrically, developers can send requests to any node (which means CockroachDB works well with load balancers). Whichever node receives the request acts as the "gateway node," as other layers process the request. + +When developers send requests to the cluster, they arrive as SQL statements, but data is ultimately written to and read from the storage layer as key-value (KV) pairs. To handle this, the SQL layer converts SQL statements into a plan of KV operations, which it passes along to the transaction layer. + +### Interactions with other layers + +In relationship to other layers in CockroachDB, the SQL layer: + +- Sends requests to the transaction layer. + +## Components + +### Relational structure + +Developers experience data stored in CockroachDB in a relational structure, i.e., rows and columns. Sets of rows and columns are organized into tables. Collections of tables are organized into databases. Your cluster can contain many databases. + +Because of this structure, CockroachDB provides typical relational features like constraints (e.g., foreign keys). This lets application developers trust that the database will ensure consistent structuring of the application's data; data validation doesn't need to be built into the application logic separately. + +### SQL API + +CockroachDB implements a large portion of the ANSI SQL standard to manifest its relational structure. You can view [all of the SQL features CockroachDB supports here](../sql-feature-support.html). + +Importantly, through the SQL API, we also let developers use ACID-semantic transactions like they would through any SQL database (`BEGIN`, `END`, `COMMIT`, etc.) + +### PostgreSQL wire protocol + +SQL queries reach your cluster through the PostgreSQL wire protocol. This makes connecting your application to the cluster simple by supporting most PostgreSQL-compatible drivers, as well as many PostgreSQL ORMs, such as GORM (Go) and Hibernate (Java). + +### SQL parser, planner, executor + +After your node ultimately receives a SQL request from a client, CockroachDB parses the statement, [creates a query plan](../cost-based-optimizer.html), and then executes the plan. + +#### Parsing + +Received queries are parsed against our `yacc` file (which describes our supported syntax), and converts the string version of each query into [abstract syntax trees](https://en.wikipedia.org/wiki/Abstract_syntax_tree) (AST). + +#### Logical planning + +The AST is subsequently transformed into a query plan in three phases: + +1. The AST is transformed into a high-level logical query plan. During this transformation, CockroachDB also performs [semantic analysis](https://en.wikipedia.org/wiki/Semantic_analysis_(compilers)), which includes checking whether the query is valid, resolving names, eliminating unneeded intermediate computations, and finalizing which data types to use for intermediate results. + +2. The logical plan is *simplified* using transformation optimizations that are always valid. + +3. The logical plan is *optimized* using a [search algorithm](../cost-based-optimizer.html) that evaluates many possible ways to execute a query and selects an execution plan with the least costs. + +The result of the optimization phase is an optimized logical plan. This can be observed with [`EXPLAIN`](../explain.html). + +#### Physical planning + +The physical planning phase decides which nodes will participate in +the execution of the query, based on range locality information. This +is where CockroachDB decides to distribute a query to perform some +computations close to where the data is stored. + +The result of physical planning is a physical plan and can be observed +with [`EXPLAIN(DISTSQL)`](../explain.html). + +#### Query execution + +Components of the physical plan are sent to one or more nodes for execution. On each node, CockroachDB spawns a *logical processor* to compute a part of the query. Logical processors inside or across nodes communicate with each other over a *logical flow* of data. The combined results of the query are sent back to the first node where the query was received, to be sent further to the SQL client. + +Each processor uses an encoded form for the scalar values manipulated by the query. This is a binary form which is different from that used in SQL. So the values listed in the SQL query must be encoded, and the data communicated between logical processors, and read from disk, must be decoded before it is sent back to the SQL client. + +#### Vectorized query execution + +If [vectorized execution](../vectorized-execution.html) is enabled, the physical plan is sent to nodes to be processed by the vectorized execution engine. + +Upon receiving the physical plan, the vectorized engine reads batches of table data [from disk](storage-layer.html) and converts the data from row format to columnar format. These batches of column data are stored in memory so the engine can access them quickly during execution. + +The vectorized engine uses specialized, precompiled functions that quickly iterate over the type-specific arrays of column data. The columnar output from the functions is stored in memory as the engine processes each column of data. + +After processing all columns of data in the input buffer, the engine converts the columnar output back to row format, and then returns the processed rows to the SQL interface. After a batch of table data has been fully processed, the engine reads the following batch of table data for processing, until the query has been executed. + +### Encoding + +Though SQL queries are written in parsable strings, lower layers of CockroachDB deal primarily in bytes. This means at the SQL layer, in query execution, CockroachDB must convert row data from their SQL representation as strings into bytes, and convert bytes returned from lower layers into SQL data that can be passed back to the client. + +It's also important––for indexed columns––that this byte encoding preserve the same sort order as the data type it represents. This is because of the way CockroachDB ultimately stores data in a sorted key-value map; storing bytes in the same order as the data it represents lets us efficiently scan KV data. + +However, for non-indexed columns (e.g., non-`PRIMARY KEY` columns), CockroachDB instead uses an encoding (known as "value encoding") which consumes less space but does not preserve ordering. + +You can find more exhaustive detail in the [Encoding Tech Note](https://github.com/cockroachdb/cockroach/blob/master/docs/tech-notes/encoding.md). + +### DistSQL + +Because CockroachDB is a distributed database, we've developed a Distributed SQL (DistSQL) optimization tool for some queries, which can dramatically speed up queries that involve many ranges. Though DistSQL's architecture is worthy of its own documentation, this cursory explanation can provide some insight into how it works. + +In non-distributed queries, the coordinating node receives all of the rows that match its query, and then performs any computations on the entire data set. + +However, for DistSQL-compatible queries, each node does computations on the rows it contains, and then sends the results (instead of the entire rows) to the coordinating node. The coordinating node then aggregates the results from each node, and finally returns a single response to the client. + +This dramatically reduces the amount of data brought to the coordinating node, and leverages the well-proven concept of parallel computing, ultimately reducing the time it takes for complex queries to complete. In addition, this processes data on the node that already stores it, which lets CockroachDB handle row-sets that are larger than an individual node's storage. + +To run SQL statements in a distributed fashion, we introduce a couple of concepts: + +- **Logical plan**: Similar to the AST/`planNode` tree described above, it represents the abstract (non-distributed) data flow through computation stages. +- **Physical plan**: A physical plan is conceptually a mapping of the logical plan nodes to physical machines running `cockroach`. Logical plan nodes are replicated and specialized depending on the cluster topology. Like `planNodes` above, these components of the physical plan are scheduled and run on the cluster. + +You can find much greater detail in the [DistSQL RFC](https://github.com/cockroachdb/cockroach/blob/master/docs/RFCS/20160421_distributed_sql.md). + +## Technical interactions with other layers + +### SQL and transaction layer + +KV operations from executed `planNodes` are sent to the transaction layer. + +## What's next? + +Learn how CockroachDB handles concurrent requests in the [transaction layer](transaction-layer.html). diff --git a/v20.2/architecture/storage-layer.md b/v20.2/architecture/storage-layer.md new file mode 100644 index 00000000000..49440c771b3 --- /dev/null +++ b/v20.2/architecture/storage-layer.md @@ -0,0 +1,87 @@ +--- +title: Storage Layer +summary: The storage layer of CockroachDB's architecture reads and writes data to disk. +toc: true +--- + +The storage layer of CockroachDB's architecture reads and writes data to disk. + +{{site.data.alerts.callout_info}} +If you haven't already, we recommend reading the [Architecture Overview](overview.html). +{{site.data.alerts.end}} + + +## Overview + +Each CockroachDB node contains at least one `store`, specified when the node starts, which is where the `cockroach` process reads and writes its data on disk. + +This data is stored as key-value pairs on disk using RocksDB, which is treated primarily as a black-box API. Internally, each store contains two instances of RocksDB: + +- One for storing temporary distributed SQL data +- One for all other data on the node + +In addition, there is also a block cache shared amongst all of the stores in a node. These stores in turn have a collection of range replicas. More than one replica for a range will never be placed on the same store or even the same node. + +### Interactions with other layers + +In relationship to other layers in CockroachDB, the storage layer: + +- Serves successful reads and writes from the replication layer. + +## Components + +### RocksDB + +CockroachDB uses RocksDB––an embedded key-value store––to read and write data to disk. You can find more information about it on the [RocksDB Basics GitHub page](https://github.com/facebook/rocksdb/wiki/RocksDB-Basics). + +RocksDB integrates really well with CockroachDB for a number of reasons: + +- Key-value store, which makes mapping to our key-value layer simple +- Atomic write batches and snapshots, which give us a subset of transactions + +Efficient storage for the keys is guaranteed by the underlying RocksDB engine by means of prefix compression. + +### MVCC + +CockroachDB relies heavily on [multi-version concurrency control (MVCC)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) to process concurrent requests and guarantee consistency. Much of this work is done by using [hybrid logical clock (HLC) timestamps](transaction-layer.html#time-and-hybrid-logical-clocks) to differentiate between versions of data, track commit timestamps, and identify a value's garbage collection expiration. All of this MVCC data is then stored in RocksDB. + +Despite being implemented in the storage layer, MVCC values are widely used to enforce consistency in the [transaction layer](transaction-layer.html). For example, CockroachDB maintains a [timestamp cache](transaction-layer.html#timestamp-cache), which stores the timestamp of the last time that the key was read. If a write operation occurs at a lower timestamp than the largest value in the read timestamp cache, it signifies there’s a potential anomaly and the transaction must be restarted at a later timestamp. + +#### Time-travel + +As described in the [SQL:2011 standard](https://en.wikipedia.org/wiki/SQL:2011#Temporal_support), CockroachDB supports time travel queries (enabled by MVCC). + +To do this, all of the schema information also has an MVCC-like model behind it. This lets you perform `SELECT...AS OF SYSTEM TIME`, and CockroachDB uses the schema information as of that time to formulate the queries. + +Using these tools, you can get consistent data from your database as far back as your garbage collection period. + +### Garbage collection + +CockroachDB regularly garbage collects MVCC values to reduce the size of data stored on disk. To do this, we compact old MVCC values when there is a newer MVCC value with a timestamp that's older than the garbage collection period. The garbage collection period can be set at the cluster, database, or table level by configuring the [`gc.ttlseconds` replication zone variable](../configure-replication-zones.html#gc-ttlseconds). For more information about replication zones, see [Configure Replication Zones](../configure-replication-zones.html). + +#### Protected timestamps + + Garbage collection can only run on MVCC values which are not covered by a *protected timestamp*. The protected timestamp subsystem exists to ensure the safety of operations that rely on historical data, such as: + +- [Imports](../import.html), including [`IMPORT INTO`](../import-into.html) +- [Backups](../backup.html) +- [Change data capture (CDC)](../change-data-capture.html) (a.k.a. [changefeeds](../create-changefeed.html)) +- [Online schema changes](../online-schema-changes.html) + +Protected timestamps ensure the safety of historical data while also enabling shorter [GC TTLs](../configure-replication-zones.html#gc-ttlseconds). A shorter GC TTL means that fewer previous MVCC values are kept around. This can help lower query execution costs for workloads which update rows frequently throughout the day, since [the SQL layer](sql-layer.html) has to scan over previous MVCC values to find the current value of a row. + +##### How protected timestamps work + +Protected timestamps work by creating *protection records*, which are stored in an internal system table. When a long-running job such as a backup wants to protect data at a certain timestamp from being garbage collected, it creates a protection record associated with that data and timestamp. + +Upon successful creation of a protection record, the MVCC values for the specified data at timestamps less than or equal to the protected timestamp will not be garbage collected. When the job that created the protection record finishes its work, it removes the record, allowing the garbage collector to run on the formerly protected values. + +## Interactions with other layers + +### Storage and replication layers + +The storage layer commits writes from the Raft log to disk, as well as returns requested data (i.e., reads) to the replication layer. + +## What's next? + +Now that you've learned about our architecture, [start a local cluster](../install-cockroachdb.html) and start [building an app with CockroachDB](../build-an-app-with-cockroachdb.html). diff --git a/v20.2/architecture/transaction-layer.md b/v20.2/architecture/transaction-layer.md new file mode 100644 index 00000000000..31450a13298 --- /dev/null +++ b/v20.2/architecture/transaction-layer.md @@ -0,0 +1,386 @@ +--- +title: Transaction Layer +summary: The transaction layer of CockroachDB's architecture implements support for ACID transactions by coordinating concurrent operations. +toc: true +--- + +The transaction layer of CockroachDB's architecture implements support for ACID transactions by coordinating concurrent operations. + +{{site.data.alerts.callout_info}} +If you haven't already, we recommend reading the [Architecture Overview](overview.html). +{{site.data.alerts.end}} + +## Overview + +Above all else, CockroachDB believes consistency is the most important feature of a database––without it, developers cannot build reliable tools, and businesses suffer from potentially subtle and hard to detect anomalies. + +To provide consistency, CockroachDB implements full support for ACID transaction semantics in the transaction layer. However, it's important to realize that *all* statements are handled as transactions, including single statements––this is sometimes referred to as "autocommit mode" because it behaves as if every statement is followed by a `COMMIT`. + +For code samples of using transactions in CockroachDB, see our documentation on [transactions](../transactions.html#sql-statements). + +Because CockroachDB enables transactions that can span your entire cluster (including cross-range and cross-table transactions), it achieves correctness using a distributed, atomic commit protocol called [Parallel Commits](#parallel-commits). + +### Writes and reads (phase 1) + +#### Writing + +When the transaction layer executes write operations, it doesn't directly write values to disk. Instead, it creates several things that help it mediate a distributed transaction: + +- **Locks** for all of a transaction’s writes, which represent a provisional, uncommitted state. CockroachDB has several different types of locking: + + - **Unreplicated Locks** are stored in an in-memory, per-node lock table by the [concurrency control](#concurrency-control) machinery. These locks are not replicated via [Raft](replication-layer.html#raft). + + - **Replicated Locks** (also known as [write intents](#write-intents)) are replicated via [Raft](replication-layer.html#raft), and act as a combination of a provisional value and an exclusive lock. They are essentially the same as standard [multi-version concurrency control (MVCC)](storage-layer.html#mvcc) values but also contain a pointer to the [transaction record](#transaction-records) stored on the cluster. + +- A **transaction record** stored in the range where the first write occurs, which includes the transaction's current state (which is either `PENDING`, `STAGING`, `COMMITTED`, or `ABORTED`). + +As write intents are created, CockroachDB checks for newer committed values. If newer committed values exist, the transaction may be restarted. If existing write intents for the same keys exist, it is resolved as a [transaction conflict](#transaction-conflicts). + +If transactions fail for other reasons, such as failing to pass a SQL constraint, the transaction is aborted. + +#### Reading + +If the transaction has not been aborted, the transaction layer begins executing read operations. If a read only encounters standard MVCC values, everything is fine. However, if it encounters any write intents, the operation must be resolved as a [transaction conflict](#transaction-conflicts). + +### Commits (phase 2) + +CockroachDB checks the running transaction's record to see if it's been `ABORTED`; if it has, it restarts the transaction. + +In the common case, it sets the transaction record's state to `STAGING`, and checks the transaction's pending write intents to see if they have succeeded (i.e., been replicated across the cluster). + +If the transaction passes these checks, CockroachDB responds with the transaction's success to the client, and moves on to the cleanup phase. At this point, the transaction is committed, and the client is free to begin sending more requests to the cluster. + +For a more detailed walkthrough of the commit protocol, see [Parallel Commits](#parallel-commits). + +### Cleanup (asynchronous phase 3) + +After the transaction has been committed, it should be marked as such, and all of the write intents should be resolved. To do this, the coordinating node––which kept a track of all of the keys it wrote––reaches out and: + +- Moves the state of the transaction record from `STAGING` to `COMMITTED`. +- Resolves the transaction's write intents to MVCC values by removing the element that points it to the transaction record. +- Deletes the write intents. + +This is simply an optimization, though. If operations in the future encounter write intents, they always check their transaction records––any operation can resolve or remove write intents by checking the transaction record's status. + +### Interactions with other layers + +In relationship to other layers in CockroachDB, the transaction layer: + +- Receives KV operations from the SQL layer. +- Controls the flow of KV operations sent to the distribution layer. + +## Technical details and components + +### Time and hybrid logical clocks + +In distributed systems, ordering and causality are difficult problems to solve. While it's possible to rely entirely on Raft consensus to maintain serializability, it would be inefficient for reading data. To optimize performance of reads, CockroachDB implements hybrid-logical clocks (HLC) which are composed of a physical component (always close to local wall time) and a logical component (used to distinguish between events with the same physical component). This means that HLC time is always greater than or equal to the wall time. You can find more detail in the [HLC paper](http://www.cse.buffalo.edu/tech-reports/2014-04.pdf). + +In terms of transactions, the gateway node picks a timestamp for the transaction using HLC time. Whenever a transaction's timestamp is mentioned, it's an HLC value. This timestamp is used to both track versions of values (through [multi-version concurrency control](storage-layer.html#mvcc)), as well as provide our transactional isolation guarantees. + +When nodes send requests to other nodes, they include the timestamp generated by their local HLCs (which includes both physical and logical components). When nodes receive requests, they inform their local HLC of the timestamp supplied with the event by the sender. This is useful in guaranteeing that all data read/written on a node is at a timestamp less than the next HLC time. + +This then lets the node primarily responsible for the range (i.e., the leaseholder) serve reads for data it stores by ensuring the transaction reading the data is at an HLC time greater than the MVCC value it's reading (i.e., the read always happens "after" the write). + +#### Max clock offset enforcement + +CockroachDB requires moderate levels of clock synchronization to preserve data consistency. For this reason, when a node detects that its clock is out of sync with at least half of the other nodes in the cluster by 80% of the maximum offset allowed (500ms by default), **it crashes immediately**. + +While [serializable consistency](https://en.wikipedia.org/wiki/Serializability) is maintained regardless of clock skew, skew outside the configured clock offset bounds can result in violations of single-key linearizability between causally dependent transactions. It's therefore important to prevent clocks from drifting too far by running [NTP](http://www.ntp.org/) or other clock synchronization software on each node. + +For more detail about the risks that large clock offsets can cause, see [What happens when node clocks are not properly synchronized?](../operational-faqs.html#what-happens-when-node-clocks-are-not-properly-synchronized) + +### Timestamp cache + +As part of providing serializability, whenever an operation reads a value, we store the operation's timestamp in a timestamp cache, which shows the high-water mark for values being read. + +Whenever a write occurs, its timestamp is checked against the timestamp cache. If the timestamp is less than the timestamp cache's latest value, we attempt to push the timestamp for its transaction forward to a later time. Pushing the timestamp might cause the transaction to restart in the second phase of the transaction (see [read refreshing](#read-refreshing)). + +### client.Txn and TxnCoordSender + +As we mentioned in the SQL layer's architectural overview, CockroachDB converts all SQL statements into key-value (KV) operations, which is how data is ultimately stored and accessed. + +All of the KV operations generated from the SQL layer use `client.Txn`, which is the transactional interface for the CockroachDB KV layer––but, as we discussed above, all statements are treated as transactions, so all statements use this interface. + +However, `client.Txn` is actually just a wrapper around `TxnCoordSender`, which plays a crucial role in our code base by: + +- Dealing with transactions' state. After a transaction is started, `TxnCoordSender` starts asynchronously sending heartbeat messages to that transaction's transaction record, which signals that it should be kept alive. If the `TxnCoordSender`'s heartbeating stops, the transaction record is moved to the `ABORTED` status. +- Tracking each written key or key range over the course of the transaction. +- Clearing the accumulated write intent for the transaction when it's committed or aborted. All requests being performed as part of a transaction have to go through the same `TxnCoordSender` to account for all of its write intents, which optimizes the cleanup process. + +After setting up this bookkeeping, the request is passed to the `DistSender` in the distribution layer. + +### Transaction records + +To track the status of a transaction's execution, we write a value called a transaction record to our key-value store. All of a transaction's write intents point back to this record, which lets any transaction check the status of any write intents it encounters. This kind of canonical record is crucial for supporting concurrency in a distributed environment. + +Transaction records are always written to the same range as the first key in the transaction, which is known by the `TxnCoordSender`. However, the transaction record itself isn't created until one of the following conditions occur: + +- The write operation commits +- The `TxnCoordSender` heartbeats the transaction +- An operation forces the transaction to abort + +Given this mechanism, the transaction record uses the following states: + +- `PENDING`: Indicates that the write intent's transaction is still in progress. +- `COMMITTED`: Once a transaction has completed, this status indicates that write intents can be treated as committed values. +- `STAGING`: Used to enable the [Parallel Commits](#parallel-commits) feature. Depending on the state of the write intents referenced by this record, the transaction may or may not be in a committed state. +- `ABORTED`: Indicates that the transaction was aborted and its values should be discarded. +- _Record does not exist_: If a transaction encounters a write intent whose transaction record doesn't exist, it uses the write intent's timestamp to determine how to proceed. If the write intent's timestamp is within the transaction liveness threshold, the write intent's transaction is treated as if it is `PENDING`, otherwise it's treated as if the transaction is `ABORTED`. + +The transaction record for a committed transaction remains until all its write intents are converted to MVCC values. + +### Write intents + +Values in CockroachDB are not written directly to the storage layer; instead values are written in a provisional state known as a "write intent." These are essentially MVCC records with an additional value added to them which identifies the transaction record to which the value belongs. They can be thought of as a combination of a replicated lock and a replicated provisional value. + +Whenever an operation encounters a write intent (instead of an MVCC value), it looks up the status of the transaction record to understand how it should treat the write intent value. If the transaction record is missing, the operation checks the write intent's timestamp and evaluates whether or not it is considered expired. + + CockroachDB manages concurrency control using a per-node, in-memory lock table. This table holds a collection of locks acquired by in-progress transactions, and incorporates information about write intents as they are discovered during evaluation. For more information, see the section below on [Concurrency control](#concurrency-control). + +#### Resolving write intents + +Whenever an operation encounters a write intent for a key, it attempts to "resolve" it, the result of which depends on the write intent's transaction record: + +- `COMMITTED`: The operation reads the write intent and converts it to an MVCC value by removing the write intent's pointer to the transaction record. +- `ABORTED`: The write intent is ignored and deleted. +- `PENDING`: This signals there is a [transaction conflict](#transaction-conflicts), which must be resolved. +- `STAGING`: This signals that the operation should check whether the staging transaction is still in progress by verifying that the transaction coordinator is still heartbeating the staging transaction’s record. If the coordinator is still heartbeating the record, the operation should wait. For more information, see [Parallel Commits](#parallel-commits). +- _Record does not exist_: If the write intent was created within the transaction liveness threshold, it's the same as `PENDING`, otherwise it's treated as `ABORTED`. + +### Concurrency control + + The *concurrency manager* sequences incoming requests and provides isolation between the transactions that issued those requests that intend to perform conflicting operations. This activity is also known as [concurrency control](https://en.wikipedia.org/wiki/Concurrency_control). + +The concurrency manager combines the operations of a *latch manager* and a *lock table* to accomplish this work: + +- The *latch manager* sequences the incoming requests and provides isolation between those requests. +- The *lock table* provides both locking and sequencing of requests (in concert with the latch manager). It is a per-node, in-memory data structure that holds a collection of locks acquired by in-progress transactions. To ensure compatibility with the existing system of [write intents](#write-intents) (a.k.a. replicated, exclusive locks), it pulls in information about these external locks as necessary when they are discovered in the course of evaluating requests. + +The concurrency manager enables support for pessimistic locking via [SQL](sql-layer.html) using the [`SELECT FOR UPDATE`](../select-for-update.html) statement. This statement can be used to increase throughput and decrease tail latency for contended operations. + +For more details about how the concurrency manager works with the latch manager and lock table, see the sections below: + +- [Concurrency manager](#concurrency-manager) +- [Lock table](#lock-table) +- [Latch manager](#latch-manager) + +#### Concurrency manager + + The concurrency manager is a structure that sequences incoming requests and provides isolation between the transactions that issued those requests that intend to perform conflicting operations. During sequencing, conflicts are discovered and any found are resolved through a combination of passive queuing and active pushing. Once a request has been sequenced, it is free to evaluate without concerns of conflicting with other in-flight requests due to the isolation provided by the manager. This isolation is guaranteed for the lifetime of the request but terminates once the request completes. + +Each request in a transaction should be isolated from other requests, both during the request's lifetime and after the request has completed (assuming it acquired locks), but within the surrounding transaction's lifetime. + +The manager accommodates this by allowing transactional requests to acquire locks, which outlive the requests themselves. Locks extend the duration of the isolation provided over specific keys to the lifetime of the lock-holder transaction itself. They are (typically) only released when the transaction commits or aborts. Other requests that find these locks while being sequenced wait on them to be released in a queue before proceeding. Because locks are checked during sequencing, locks don't need to be checked again during evaluation. + +However, not all locks are stored directly under the manager's control, so not all locks are discoverable during sequencing. Specifically, write intents (replicated, exclusive locks) are stored inline in the MVCC keyspace, so they are not detectable until request evaluation time. To accommodate this form of lock storage, the manager integrates information about external locks with the concurrency manager structure. + +{{site.data.alerts.callout_info}} +Currently, the concurrency manager operates on an unreplicated lock table structure. In the future, we intend to pull all locks, including those associated with [write intents](#write-intents), into the concurrency manager directly through a replicated lock table structure. +{{site.data.alerts.end}} + +Fairness is ensured between requests. In general, if any two requests conflict then the request that arrived first will be sequenced first. As such, sequencing guarantees first-in, first-out (FIFO) semantics. The primary exception to this is that a request that is part of a transaction which has already acquired a lock does not need to wait on that lock during sequencing, and can therefore ignore any queue that has formed on the lock. For other exceptions to this sequencing guarantee, see the [lock table](#lock-table) section below. + +#### Lock table + + The lock table is a per-node, in-memory data structure that holds a collection of locks acquired by in-progress transactions. Each lock in the table has a possibly-empty lock wait-queue associated with it, where conflicting transactions can queue while waiting for the lock to be released. Items in the locally stored lock wait-queue are propagated as necessary (via RPC) to the existing [`TxnWaitQueue`](#txnwaitqueue), which is stored on the leader of the range's Raft group that contains the [transaction record](#transaction-records). + +The database is read and written using "requests". Transactions are composed of one or more requests. Isolation is needed across requests. Additionally, since transactions represent a group of requests, isolation is needed across such groups. Part of this isolation is accomplished by maintaining multiple versions and part by allowing requests to acquire locks. Even the isolation based on multiple versions requires some form of mutual exclusion to ensure that a read and a conflicting lock acquisition do not happen concurrently. The lock table provides both locking and sequencing of requests (in concert with the use of latches). + +Locks outlive the requests themselves and thereby extend the duration of the isolation provided over specific keys to the lifetime of the lock-holder transaction itself. They are (typically) only released when the transaction commits or aborts. Other requests that find these locks while being sequenced wait on them to be released in a queue before proceeding. Because locks are checked during sequencing, requests are guaranteed access to all declared keys after they have been sequenced. In other words, locks don't need to be checked again during evaluation. + +{{site.data.alerts.callout_info}} +Currently, not all locks are stored directly under lock table control. Some locks are stored as [write intents](#write-intents) in the MVCC layer, and are thus not discoverable during sequencing. Specifically, write intents (replicated, exclusive locks) are stored inline in the MVCC keyspace, so they are often not detectable until request evaluation time. To accommodate this form of lock storage, the lock table adds information about these locks as they are encountered during evaluation. In the future, we intend to pull all locks, including those associated with write intents, into the lock table directly. +{{site.data.alerts.end}} + +The lock table also provides fairness between requests. If two requests conflict then the request that arrived first will typically be sequenced first. There are some exceptions: + +- A request that is part of a transaction which has already acquired a lock does not need to wait on that lock during sequencing, and can therefore ignore any queue that has formed on the lock. + +- Contending requests that encounter different levels of contention may be sequenced in non-FIFO order. This is to allow for greater concurrency. For example, if requests *R1* and *R2* contend on key *K2*, but *R1* is also waiting at key *K1*, *R2* may slip past *R1* and evaluate. + +#### Latch manager + +The latch manager sequences incoming requests and provides isolation between those requests under the supervision of the [concurrency manager](#concurrency-manager). + +The way the latch manager works is as follows: + +As write requests occur for a range, the range's leaseholder serializes them; that is, they are placed into some consistent order. + +To enforce this serialization, the leaseholder creates a "latch" for the keys in the write value, providing uncontested access to the keys. If other requests come into the leaseholder for the same set of keys, they must wait for the latch to be released before they can proceed. + +Read requests also generate latches. Multiple read latches over the same keys can be held concurrently, but a read latch and a write latch over the same keys cannot. + +Another way to think of a latch is like a [mutex](https://en.wikipedia.org/wiki/Lock_(computer_science)) which is only needed for the duration of a single, low-level request. To coordinate longer-running, higher-level requests (i.e., client transactions), we use a durable system of [write intents](#write-intents). + +### Isolation levels + +Isolation is an element of [ACID transactions](https://en.wikipedia.org/wiki/ACID), which determines how concurrency is controlled, and ultimately guarantees consistency. + +CockroachDB executes all transactions at the strongest ANSI transaction isolation level: `SERIALIZABLE`. All other ANSI transaction isolation levels (e.g., `SNAPSHOT`, `READ UNCOMMITTED`, `READ COMMITTED`, and `REPEATABLE READ`) are automatically upgraded to `SERIALIZABLE`. Weaker isolation levels have historically been used to maximize transaction throughput. However, [recent research](http://www.bailis.org/papers/acidrain-sigmod2017.pdf) has demonstrated that the use of weak isolation levels results in substantial vulnerability to concurrency-based attacks. + +CockroachDB now only supports `SERIALIZABLE` isolation. In previous versions of CockroachDB, you could set transactions to `SNAPSHOT` isolation, but that feature has been removed. + +`SERIALIZABLE` isolation does not allow any anomalies in your data, and is enforced by requiring the client to retry transactions if serializability violations are possible. + +### Transaction conflicts + +CockroachDB's transactions allow the following types of conflicts that involve running into an intent: + +- **Write/write**, where two `PENDING` transactions create write intents for the same key. +- **Write/read**, when a read encounters an existing write intent with a timestamp less than its own. + +To make this simpler to understand, we'll call the first transaction `TxnA` and the transaction that encounters its write intents `TxnB`. + +CockroachDB proceeds through the following steps: + +1. If the transaction has an explicit priority set (i.e., `HIGH` or `LOW`), the transaction with the lower priority is aborted (in the write/write case) or has its timestamp pushed (in the write/read case). + +1. If the encountered transaction is expired, it's `ABORTED` and conflict resolution succeeds. We consider a write intent expired if: + - It doesn't have a transaction record and its timestamp is outside of the transaction liveness threshold. + - Its transaction record hasn't been heartbeated within the transaction liveness threshold. + +2. `TxnB` enters the `TxnWaitQueue` to wait for `TxnA` to complete. + +Additionally, the following types of conflicts that do not involve running into intents can arise: + +- **Write after read**, when a write with a lower timestamp encounters a later read. This is handled through the [timestamp cache](#timestamp-cache). +- **Read within uncertainty window**, when a read encounters a value with a higher timestamp but it's ambiguous whether the value should be considered to be in the future or in the past of the transaction because of possible *clock skew*. This is handled by attempting to push the transaction's timestamp beyond the uncertain value (see [read refreshing](#read-refreshing)). Note that, if the transaction has to be retried, reads will never encounter uncertainty issues on any node which was previously visited, and that there's never any uncertainty on values read from the transaction's gateway node. + +### TxnWaitQueue + +The `TxnWaitQueue` tracks all transactions that could not push a transaction whose writes they encountered, and must wait for the blocking transaction to complete before they can proceed. + +The `TxnWaitQueue`'s structure is a map of blocking transaction IDs to those they're blocking. For example: + +~~~ +txnA -> txn1, txn2 +txnB -> txn3, txn4, txn5 +~~~ + +Importantly, all of this activity happens on a single node, which is the leader of the range's Raft group that contains the transaction record. + +Once the transaction does resolve––by committing or aborting––a signal is sent to the `TxnWaitQueue`, which lets all transactions that were blocked by the resolved transaction begin executing. + +Blocked transactions also check the status of their own transaction to ensure they're still active. If the blocked transaction was aborted, it's simply removed. + +If there is a deadlock between transactions (i.e., they're each blocked by each other's Write Intents), one of the transactions is randomly aborted. In the above example, this would happen if `TxnA` blocked `TxnB` on `key1` and `TxnB` blocked `TxnA` on `key2`. + +### Read refreshing + +Whenever a transaction's timestamp has been pushed, additional checks are required before allowing it to commit at the pushed timestamp: any values which the transaction previously read must be checked to verify that no writes have subsequently occurred between the original transaction timestamp and the pushed transaction timestamp. This check prevents serializability violation. The check is done by keeping track of all the reads using a dedicated `RefreshRequest`. If this succeeds, the transaction is allowed to commit (transactions perform this check at commit time if they've been pushed by a different transaction or by the timestamp cache, or they perform the check whenever they encounter a `ReadWithinUncertaintyIntervalError` immediately, before continuing). +If the refreshing is unsuccessful, then the transaction must be retried at the pushed timestamp. + +### Transaction pipelining + +Transactional writes are pipelined when being replicated and when being written to disk, dramatically reducing the latency of transactions that perform multiple writes. For example, consider the following transaction: + +{% include copy-clipboard.html %} +~~~ sql +-- CREATE TABLE kv (id UUID PRIMARY KEY DEFAULT gen_random_uuid(), key VARCHAR, value VARCHAR); +> BEGIN; +INSERT into kv (key, value) VALUES ('apple', 'red'); +INSERT into kv (key, value) VALUES ('banana', 'yellow'); +INSERT into kv (key, value) VALUES ('orange', 'orange'); +COMMIT; +~~~ + +With transaction pipelining, write intents are replicated from leaseholders in parallel, so the waiting all happens at the end, at transaction commit time. + +At a high level, transaction pipelining works as follows: + +1. For each statement, the transaction gateway node communicates with the leaseholders (*L*1, *L*2, *L*3, ..., *L*i) for the ranges it wants to write to. Since the primary keys in the table above are UUIDs, the ranges are probably split across multiple leaseholders (this is a good thing, as it decreases [transaction conflicts](#transaction-conflicts)). + +2. Each leaseholder *L*i receives the communication from the transaction gateway node and does the following in parallel: + - Creates write intents and sends them to its follower nodes. + - Responds to the transaction gateway node that the write intents have been sent. Note that replication of the intents is still in-flight at this stage. + +3. When attempting to commit, the transaction gateway node then waits for the write intents to be replicated in parallel to all of the leaseholders' followers. When it receives responses from the leaseholders that the write intents have propagated, it commits the transaction. + +In terms of the SQL snippet shown above, all of the waiting for write intents to propagate and be committed happens once, at the very end of the transaction, rather than for each individual write. This means that the cost of multiple writes is not `O(n)` in the number of SQL DML statements; instead, it's `O(1)`. + +### Parallel Commits + +The *Parallel Commits* feature introduces a new, optimized atomic commit protocol that cuts the commit latency of a transaction in half, from two rounds of consensus down to one. Combined with [Transaction pipelining](#transaction-pipelining), this brings the latency incurred by common OLTP transactions to near the theoretical minimum: the sum of all read latencies plus one round of consensus latency. + +Under the new atomic commit protocol, the transaction coordinator can return to the client eagerly when it knows that the writes in the transaction have succeeded. Once this occurs, the transaction coordinator can set the transaction record's state to `COMMITTED` and resolve the transaction's write intents asynchronously. + +The transaction coordinator is able to do this while maintaining correctness guarantees because it populates the transaction record with enough information (via a new `STAGING` state, and an array of in-flight writes) for other transactions to determine whether all writes in the transaction are present, and thus prove whether or not the transaction is committed. + +For an example showing how the Parallel Commits feature works in more detail, see [Parallel Commits - step by step](#parallel-commits-step-by-step). + +{{site.data.alerts.callout_info}} +The latency until intents are resolved is unchanged by the introduction of Parallel Commits: two rounds of consensus are still required to resolve intents. This means that [contended workloads](../performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) are expected to profit less from this feature. +{{site.data.alerts.end}} + +#### Parallel Commits - step by step + +This section contains a step by step example of a transaction that writes its data using the Parallel Commits atomic commit protocol and does not encounter any errors or conflicts. + +##### Step 1 + +The client starts the transaction. A transaction coordinator is created to manage the state of that transaction. + +![parallel-commits-00.png](../../images/{{page.version.version}}/parallel-commits-00.png "Parallel Commits Diagram #1") + +##### Step 2 + +The client issues a write to the "Apple" key. The transaction coordinator begins the process of laying down a write intent on the key where the data will be written. The write intent has a timestamp and a pointer to an as-yet nonexistent transaction record. Additionally, each write intent in the transaction is assigned a unique sequence number which is used to uniquely identify it. + +The coordinator avoids creating the record for as long as possible in the transaction's lifecycle as an optimization. The fact that the transaction record does not yet exist is denoted in the diagram by its dotted lines. + +{{site.data.alerts.callout_info}} +The coordinator does not need to wait for write intents to replicate from leaseholders before moving on to the next statement from the client, since that is handled in parallel by [Transaction Pipelining](#transaction-pipelining). +{{site.data.alerts.end}} + +![parallel-commits-01.png](../../images/{{page.version.version}}/parallel-commits-01.png "Parallel Commits Diagram #2") + +##### Step 3 + +The client issues a write to the "Berry" key. The transaction coordinator lays down a write intent on the key where the data will be written. This write intent has a pointer to the same transaction record as the intent created in [Step 2](#step-2), since these write intents are part of the same transaction. + +As before, the coordinator does not need to wait for write intents to replicate from leaseholders before moving on to the next statement from the client. + +![parallel-commits-02.png](../../images/{{page.version.version}}/parallel-commits-02.png "Parallel Commits Diagram #3") + +##### Step 4 + +The client issues a request to commit the transaction's writes. The transaction coordinator creates the transaction record and immediately sets the record's state to `STAGING`, and records the keys of each write that the transaction has in flight. + +It does this without waiting to see whether the writes from Steps [2](#step-2) and [3](#step-3) have succeeded. + +![parallel-commits-03.png](../../images/{{page.version.version}}/parallel-commits-03.png "Parallel Commits Diagram #4") + +##### Step 5 + +The transaction coordinator, having received the client's `COMMIT` request, waits for the pending writes to succeed (i.e., be replicated across the cluster). Once all of the pending writes have succeeded, the coordinator returns a message to the client, letting it know that its transaction has committed successfully. + +![parallel-commits-04.png](../../images/{{page.version.version}}/parallel-commits-04.png "Parallel Commits Diagram #4") + +The transaction is now considered atomically committed, even though the state of its transaction record is still `STAGING`. The reason this is still considered an atomic commit condition is that a transaction is considered committed if it is one of the following logically equivalent states: + +1. The transaction record's state is `STAGING`, and its list of pending writes have all succeeded (i.e., the `InFlightWrites` have achieved consensus across the cluster). Any observer of this transaction can verify that its writes have replicated. Transactions in this state are *implicitly committed*. + +2. The transaction record's state is `COMMITTED`. Transactions in this state are *explicitly committed*. + +Despite their logical equivalence, the transaction coordinator now works as quickly as possible to move the transaction record from the `STAGING` to the `COMMITTED` state so that other transactions do not encounter a possibly conflicting transaction in the `STAGING` state and then have to do the work of verifying that the staging transaction's list of pending writes has succeeded. Doing that verification (also known as the "transaction status recovery process") would be slow. + +Additionally, when other transactions encounter a transaction in `STAGING` state, they check whether the staging transaction is still in progress by verifying that the transaction coordinator is still heartbeating that staging transaction’s record. If the coordinator is still heartbeating the record, the other transactions will wait, on the theory that letting the coordinator update the transaction record with the final result of the attempt to commit will be faster than going through the transaction status recovery process. This means that in practice, the transaction status recovery process is only used if the transaction coordinator dies due to an untimely crash. + +## Technical interactions with other layers + +### Transaction and SQL layer + +The transaction layer receives KV operations from `planNodes` executed in the SQL layer. + +### Transaction and distribution layer + +The `TxnCoordSender` sends its KV requests to `DistSender` in the distribution layer. + +## What's next? + +Learn how CockroachDB presents a unified view of your cluster's data in the [distribution layer](distribution-layer.html). + + + +[storage]: storage-layer.html +[sql]: sql-layer.html diff --git a/v20.2/array.md b/v20.2/array.md new file mode 100644 index 00000000000..900fef73bea --- /dev/null +++ b/v20.2/array.md @@ -0,0 +1,281 @@ +--- +title: ARRAY +summary: The ARRAY data type stores one-dimensional, 1-indexed, homogeneous arrays of any non-array data types. +toc: true +--- + +The `ARRAY` data type stores one-dimensional, 1-indexed, homogeneous arrays of any non-array [data type](data-types.html). + +The `ARRAY` data type is useful for ensuring compatibility with ORMs and other tools. However, if such compatibility is not a concern, it's more flexible to design your schema with normalized tables. + + CockroachDB supports indexing array columns with [inverted indexes](inverted-indexes.html). This permits accelerating containment queries ([`@>`](functions-and-operators.html#operator-contains) and [`<@`](functions-and-operators.html#operator-is-contained-by)) on array columns by adding an index to them. + +{{site.data.alerts.callout_info}} +CockroachDB does not support nested arrays or ordering by arrays. +{{site.data.alerts.end}} + +{% include {{page.version.version}}/sql/vectorized-support.md %} + +## Syntax + +A value of data type `ARRAY` can be expressed in the following ways: + +- Appending square brackets (`[]`) to any non-array [data type](data-types.html). +- Adding the term `ARRAY` to any non-array [data type](data-types.html). + +## Size + +The size of an `ARRAY` value is variable, but it's recommended to keep values under 1 MB to ensure performance. Above that threshold, [write amplification](https://en.wikipedia.org/wiki/Write_amplification) and other considerations may cause significant performance degradation. + +## Examples + +{{site.data.alerts.callout_success}} +For a complete list of array functions built into CockroachDB, see the [documentation on array functions](functions-and-operators.html#array-functions). +{{site.data.alerts.end}} + +### Creating an array column by appending square brackets + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE a (b STRING[]); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO a VALUES (ARRAY['sky', 'road', 'car']); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM a; +~~~ + +~~~ ++----------------------+ +| b | ++----------------------+ +| {"sky","road","car"} | ++----------------------+ +(1 row) +~~~ + +### Creating an array column by adding the term `ARRAY` + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE c (d INT ARRAY); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO c VALUES (ARRAY[10,20,30]); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++------------+ +| d | ++------------+ +| {10,20,30} | ++------------+ +(1 row) +~~~ + +### Accessing an array element using array index + +{{site.data.alerts.callout_info}} +Arrays in CockroachDB are 1-indexed. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++------------+ +| d | ++------------+ +| {10,20,30} | ++------------+ +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT d[2] FROM c; +~~~ + +~~~ ++------+ +| d[2] | ++------+ +| 20 | ++------+ +(1 row) +~~~ + +### Appending an element to an array + +#### Using the `array_append` function + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++------------+ +| d | ++------------+ +| {10,20,30} | ++------------+ +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE c SET d = array_append(d, 40) WHERE d[3] = 30; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++---------------+ +| d | ++---------------+ +| {10,20,30,40} | ++---------------+ +(1 row) +~~~ + +#### Using the append (`||`) operator + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++---------------+ +| d | ++---------------+ +| {10,20,30,40} | ++---------------+ +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE c SET d = d || 50 WHERE d[4] = 40; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM c; +~~~ + +~~~ ++------------------+ +| d | ++------------------+ +| {10,20,30,40,50} | ++------------------+ +(1 row) +~~~ + +## Supported casting and conversion + +[Casting](data-types.html#data-type-conversions-and-casts) between `ARRAY` values is supported when the data types of the arrays support casting. For example, it is possible to cast from a `BOOL` array to an `INT` array but not from a `BOOL` array to a `TIMESTAMP` array: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT ARRAY[true,false,true]::INT[]; +~~~ + +~~~ + array ++---------+ + {1,0,1} +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT ARRAY[true,false,true]::TIMESTAMP[]; +~~~ + +~~~ +pq: invalid cast: bool[] -> TIMESTAMP[] +~~~ + +You can cast an array to a `STRING` value, for compatibility with PostgreSQL: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT ARRAY[1,NULL,3]::string; +~~~ + +~~~ + array ++------------+ + {1,NULL,3} +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT ARRAY[(1,'a b'),(2,'c"d')]::string; +~~~ + +~~~ + array ++----------------------------------+ + {"(1,\"a b\")","(2,\"c\"\"d\")"} +(1 row) +~~~ + +### Implicit casting to `INT` and `DECIMAL` `ARRAY`s + + CockroachDB supports implicit casting from string literals to [`INT`](int.html) and [`DECIMAL`](decimal.html) `ARRAY`s, where appropriate. + +For example, if you create a table with a column of type `INT[]`: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE x (a UUID DEFAULT gen_random_uuid() PRIMARY KEY, b INT[]); +~~~ + +And then insert a string containing a comma-delimited set of integers contained in brackets: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO x(b) VALUES ('{1,2,3}'), (ARRAY[4,5,6]); +~~~ + +CockroachDB implicitly casts the string literal as an `INT[]`: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM x; +~~~ + +~~~ + a | b +---------------------------------------+---------- + 2ec0ed91-8a82-4f2e-888e-ae86ece4fc60 | {4,5,6} + a521d6e9-3a2a-490d-968c-1365cace038a | {1,2,3} +(2 rows) +~~~ + +## See also + +- [Data Types](data-types.html) +- [Inverted Indexes](inverted-indexes.html) diff --git a/v20.2/as-of-system-time.md b/v20.2/as-of-system-time.md new file mode 100644 index 00000000000..5a2e8b9062d --- /dev/null +++ b/v20.2/as-of-system-time.md @@ -0,0 +1,192 @@ +--- +title: AS OF SYSTEM TIME +summary: The AS OF SYSTEM TIME clause executes a statement as of a specified time. +toc: true +--- + +The `AS OF SYSTEM TIME timestamp` clause causes statements to execute +using the database contents "as of" a specified time in the past. + +This clause can be used to read historical data (also known as "[time +travel queries](https://www.cockroachlabs.com/blog/time-travel-queries-select-witty_subtitle-the_future/)") and can also be advantageous for performance as it decreases +transaction conflicts. For more details, see [SQL Performance Best +Practices](performance-best-practices-overview.html#use-as-of-system-time-to-decrease-conflicts-with-long-running-queries). + +{{site.data.alerts.callout_info}} +Historical data is available only within the garbage collection window, which is determined by the `ttlseconds` field in the [replication zone configuration](configure-replication-zones.html). +{{site.data.alerts.end}} + +## Synopsis + +The `AS OF SYSTEM TIME` clause is supported in multiple SQL contexts, +including but not limited to: + +- In [`SELECT` clauses](select-clause.html), at the very end of the `FROM` sub-clause. +- In [`BACKUP`](backup.html), after the parameters of the `TO` sub-clause. +- In [`RESTORE`](restore.html), after the parameters of the `FROM` sub-clause. +- In [`BEGIN`](begin-transaction.html), after the `BEGIN` keyword. +- In [`SET`](set-transaction.html), after the `SET TRANSACTION` keyword. + +## Parameters + +The `timestamp` argument supports the following formats: + +Format | Notes +---|--- +[`INT`](int.html) | Nanoseconds since the Unix epoch. +negative [`INTERVAL`](interval.html) | Added to `statement_timestamp()`, and thus must be negative. +[`STRING`](string.html) | A [`TIMESTAMP`](timestamp.html), [`INT`](int.html) of nanoseconds, or negative [`INTERVAL`](interval.html). +`experimental_follower_read_timestamp()`| A [function](functions-and-operators.html) that runs your queries at a time as close as possible to the present time known as the [follower read timestamp](follower-reads.html#run-queries-that-use-follower-reads), while remaining safe for [follower reads](follower-reads.html). + +## Examples + +### Select historical data (time-travel) + +Imagine this example represents the database's current data: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT name, balance + FROM accounts + WHERE name = 'Edna Barath'; +~~~ +~~~ ++-------------+---------+ +| name | balance | ++-------------+---------+ +| Edna Barath | 750 | +| Edna Barath | 2200 | ++-------------+---------+ +~~~ + +We could instead retrieve the values as they were on October 3, 2016 at 12:45 UTC: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT name, balance + FROM accounts + AS OF SYSTEM TIME '2016-10-03 12:45:00' + WHERE name = 'Edna Barath'; +~~~ +~~~ ++-------------+---------+ +| name | balance | ++-------------+---------+ +| Edna Barath | 450 | +| Edna Barath | 2000 | ++-------------+---------+ +~~~ + + +### Using different timestamp formats + +Assuming the following statements are run at `2016-01-01 12:00:00`, they would execute as of `2016-01-01 08:00:00`: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM t AS OF SYSTEM TIME '2016-01-01 08:00:00' +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM t AS OF SYSTEM TIME 1451635200000000000 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM t AS OF SYSTEM TIME '1451635200000000000' +~~~ + +{% include copy-clipboard.html %} +~~~sql +> SELECT * FROM t AS OF SYSTEM TIME '-4h' +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM t AS OF SYSTEM TIME INTERVAL '-4h' +~~~ + +### Selecting from multiple tables + +{{site.data.alerts.callout_info}} +It is not yet possible to select from multiple tables at different timestamps. The entire query runs at the specified time in the past. +{{site.data.alerts.end}} + +When selecting over multiple tables in a single `FROM` clause, the `AS +OF SYSTEM TIME` clause must appear at the very end and applies to the +entire `SELECT` clause. + +For example: + +{% include copy-clipboard.html %} +~~~sql +> SELECT * FROM t, u, v AS OF SYSTEM TIME '-4h'; +~~~ + +{% include copy-clipboard.html %} +~~~sql +> SELECT * FROM t JOIN u ON t.x = u.y AS OF SYSTEM TIME '-4h'; +~~~ + +{% include copy-clipboard.html %} +~~~sql +> SELECT * FROM (SELECT * FROM t), (SELECT * FROM u) AS OF SYSTEM TIME '-4h'; +~~~ + +### Using `AS OF SYSTEM TIME` in subqueries + +To enable time travel, the `AS OF SYSTEM TIME` clause must appear in +at least the top-level statement. It is not valid to use it only in a +[subquery](subqueries.html). + +For example, the following is invalid: + +~~~ +SELECT * FROM (SELECT * FROM t AS OF SYSTEM TIME '-4h'), u +~~~ + +To facilitate the composition of larger queries from simpler queries, +CockroachDB allows `AS OF SYSTEM TIME` in sub-queries under the +following conditions: + +- The top level query also specifies `AS OF SYSTEM TIME`. +- All the `AS OF SYSTEM TIME` clauses specify the same timestamp. + +For example: + +{% include copy-clipboard.html %} +~~~sql +> SELECT * FROM (SELECT * FROM t AS OF SYSTEM TIME '-4h') tp + JOIN u ON tp.x = u.y + AS OF SYSTEM TIME '-4h' -- same timestamp as above - OK. + WHERE x < 123; +~~~ + +### Using `AS OF SYSTEM TIME` in transactions + +You can use the [`BEGIN`](begin-transaction.html) statement to execute the transaction using the database contents "as of" a specified time in the past. + +{% include {{ page.version.version }}/sql/begin-transaction-as-of-system-time-example.md %} + +Alternatively, you can use the [`SET`](set-transaction.html) statement to execute the transaction using the database contents "as of" a specified time in the past. + +{% include {{ page.version.version }}/sql/set-transaction-as-of-system-time-example.md %} + +## See also + +- [Select Historical Data](select-clause.html#select-historical-data-time-travel) +- [Time-Travel Queries](https://www.cockroachlabs.com/blog/time-travel-queries-select-witty_subtitle-the_future/) +- [Follower Reads](follower-reads.html) +- [Follower Reads Topology Pattern](topology-follower-reads.html) + +## Tech note + +{{site.data.alerts.callout_info}} +Although the following format is supported, it is not intended to be used by most users. +{{site.data.alerts.end}} + +HLC timestamps can be specified using a [`DECIMAL`](decimal.html). The +integer part is the wall time in nanoseconds. The fractional part is +the logical counter, a 10-digit integer. This is the same format as +produced by the `cluster_logical_timestamp()` function. diff --git a/v20.2/authentication.md b/v20.2/authentication.md new file mode 100644 index 00000000000..ef8ad7448b7 --- /dev/null +++ b/v20.2/authentication.md @@ -0,0 +1,268 @@ +--- +title: Authentication +summary: Learn about the authentication features for secure CockroachDB clusters. +toc: true +--- + +Authentication refers to the act of verifying the identity of the other party in communication. CockroachDB requires TLS 1.2 digital certificates for inter-node and client-node authentication, which require a Certificate Authority (CA) as well as keys and certificates for nodes, clients, and (optionally) the Admin UI. This document discusses how CockroachDB uses digital certificates and also gives [conceptual overview](#background-on-public-key-cryptography-and-digital-certificates) of public key cryptography and digital certificates. + +- If you are familiar with public key cryptography and digital certificates, then reading the [Using digital certificates with CockroachDB](#using-digital-certificates-with-cockroachdb) section should be enough. +- If you are unfamiliar with public key cryptography and digital certificates, you might want to skip over to the [conceptual overview](#background-on-public-key-cryptography-and-digital-certificates) first and then come back to the [Using digital certificates with CockroachDB](#using-digital-certificates-with-cockroachdb) section. +- If you want to know how to create CockroachDB security certificates, see [Create Security Certificates](cockroach-cert.html). + +## Using digital certificates with CockroachDB + +CockroachDB uses both TLS 1.2 server and client certificates. Each CockroachDB node in a secure cluster must have a **node certificate**, which is a TLS 1.2 server certificate. Note that the node certificate is multi-functional, which means that the same certificate is presented irrespective of whether the node is acting as a server or a client. The nodes use these certificates to establish secure connections with clients and with other nodes. Node certificates have the following requirements: + +- The hostname or address (IP address or DNS name) used to reach a node, either directly or through a load balancer, must be listed in the **Common Name** or **Subject Alternative Names** fields of the certificate: + + - The values specified in [`--listen-addr`](cockroach-start.html#networking) and [`--advertise-addr`](cockroach-start.html#networking) flags, or the node hostname and fully qualified hostname if not specified + - Any host addresses/names used to reach a specific node + - Any load balancer addresses/names or DNS aliases through which the node could be reached + - `localhost` and local address if connections are made through the loopback device on the same host + +- CockroachDB must be configured to trust the certificate authority that signed the certificate. + +Based on your security setup, you can use the [`cockroach cert` commands](cockroach-cert.html), [`openssl` commands](create-security-certificates-openssl.html), or a [custom CA](create-security-certificates-custom-ca.html) to generate all the keys and certificates. + +A CockroachDB cluster consists of multiple nodes and clients. The nodes can communicate with each other, with the SQL clients, and the Admin UI. In client-node SQL communication and client-UI communication, the node acts as a server, but in inter-node communication, a node may act as a server or a client. Hence authentication in CockroachDB involves: + +- Node authentication using [TLS 1.2](https://en.wikipedia.org/wiki/Transport_Layer_Security) digital certificates. +- Client authentication using TLS digital certificates, passwords, or [GSSAPI authentication](gssapi_authentication.html) (for Enterprise users). + +### Node authentication + +To set up a secure cluster without using an existing certificate authority, you'll need to generate the following files: + +- CA certificate +- Node certificate and key +- (Optional) UI certificate and key + +### Client authentication + +CockroachDB offers three methods for client authentication: + +- **Client certificate and key authentication**, which is available to all users. To ensure the highest level of security, we recommend only using client certificate and key authentication. + + Example: + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs --user=jpointsman + ~~~ + +- **Password authentication**, which is available to users and roles who you've created passwords for. Password creation is supported only in secure clusters. + + Example: + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs --user=jpointsman + ~~~ + + Note that the client still needs the CA certificate to validate the nodes' certificates. + +- [**GSSAPI authentication**](gssapi_authentication.html), which is available to [Enterprise users](enterprise-licensing.html). + +### Using `cockroach cert` or `openssl` commands + +You can use the [`cockroach cert` commands](cockroach-cert.html) or [`openssl` commands](create-security-certificates-openssl.html) to create the CA certificate and key, and node and client certificates and keys. + +Note that the node certificate created using `cockroach cert` or`openssl` is multi-functional, which means that the same certificate is presented irrespective of whether the node is acting as a server or a client. Thus all nodes must have the following: + +- `CN=node` for the special user `node` when the node acts as a client. +- All IP addresses and DNS names for the node must be listed in the `Subject Alternative Name` field for when the node acts as a server. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate). + +**Node key and certificates** + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`node.crt` | Server certificate created using the `cockroach cert` command.

`node.crt` must have `CN=node` and the list of IP addresses and DNS names listed in the `Subject Alternative Name` field. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate).

Must be signed by the CA represented by `ca.crt`. +`node.key` | Server key created using the `cockroach cert` command. + +**Client key and certificates** + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`).

Must be signed by the CA represented by `ca.crt`. +`client..key` | Client key created using the `cockroach cert` command. + +Alternatively, you can use [password authentication](#client-authentication). Remember, the client still needs `ca.crt` for node authentication. + +### Using a custom CA + +In the previous section, we discussed the scenario where the node and client certificates are signed by the CA created using the `cockroach cert` command. But what if you want to use an external CA, like your organizational CA or a public CA? In that case, our certificates might need some modification. Here’s why: + +As mentioned earlier, the node certificate is multi-functional, as in the same certificate is presented irrespective of whether the node is acting as a server or client. To make the certificate multi-functional, the `node.crt` must have `CN=node` and the list of IP addresses and DNS names listed in the`Subject Alternative Names` field. + +But some CAs will not sign a certificate containing a `CN` that is not an IP address or domain name. Here's why: The TLS client certificates are used to authenticate the client connecting to a server. Because most client certificates authenticate a user instead of a device, the certificates contain usernames instead of hostnames. This makes it difficult for public CAs to verify the client's identity and hence most public CAs will not sign a client certificate. + +To get around this issue, we can split the node key and certificate into two: + +- `node.crt` and `node.key`: The node certificate to be presented when the node acts as a server and the corresponding key. `node.crt` must have the list of IP addresses and DNS names listed in the `Subject Alternative Names`. +- `client.node.crt` and `client.node.key`: The node certificate to be presented when the node acts as a client for another node, and the corresponding key. `client.node.crt` must have `CN=node`. + +**Node key and certificates** + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate issued by the public CA or your organizational CA. +`node.crt` | Node certificate for when node acts as server.

All IP addresses and DNS names for the node must be listed in the `Subject Alternative Name`. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate).

Must be signed by the CA represented by `ca.crt`. +`node.key` | Server key corresponding to `node.crt`. +`client.node.crt` | Node certificate for when node acts as client.

Must have `CN=node`.

Must be signed by the CA represented by `ca.crt`. +`client.node.key` | Client key corresponding to `client.node.crt`. + +Optionally, if you have a certificate issued by a public CA to securely access the Admin UI, you need to place the certificate and key (`ui.crt` and `ui.key` respectively) in the directory specified by the `--certs-dir` flag. + +**Client key and certificates** + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate issued by the public CA or your organizational CA. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`).

Must be signed by the CA represented by `ca.crt`. +`client..key` | Client key corresponding to `client..crt`. + +Alternatively, you can use [password authentication](#client-authentication). Remember, the client still needs `ca.crt` for node authentication. + +### Using a public CA certificate to access the Admin UI for a secure cluster + +One of the limitations of using `cockroach cert` or `openssl` is that the browsers used to access the CockroachDB Admin UI do not trust the node certificates presented to them. Web browsers come preloaded with CA certificates from well-established entities (e.g., GlobalSign and DigiTrust). The CA certificate generated using the `cockroach cert` or `openssl` is not preloaded in the browser. Hence on accessing the Admin UI for a secure cluster, you get the “Unsafe page” warning. Now you could add the CA certificate to the browser to avoid the warning, but that is not a recommended practice. Instead, you can use the established CAs (for example, Let’s Encrypt), to create a certificate and key to access the Admin UI. + +Once you have the UI cert and key, add it to the Certificates directory specified by the `--certs-dir` flag in the `cockroach cert` command. The next time the browser tries to access the UI, the node will present the UI cert instead of the node cert, and you’ll not see the “unsafe site” warning anymore. + +**Node key and certificates** + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`node.crt` | Server certificate created using the `cockroach cert` command.

`node.crt` must have `CN=node` and the list of IP addresses and DNS names listed in the `Subject Alternative Name` field. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate).

Must be signed by the CA represented by `ca.crt`. +`node.key` | Server key created using the `cockroach cert` command. +`ui.crt` | UI certificate signed by the public CA. `ui.crt` must have the IP addresses and DNS names used to reach the Admin UI listed in the `Subject Alternative Name`. +`ui.key` | UI key corresponding to `ui.crt`. + +**Client key and certificates** + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`).

Must be signed by the CA represented by `ca.crt`. +`client..key` | Client key created using the `cockroach cert` command. + +Alternatively, you can use [password authentication](#client-authentication). Remember, the client still needs `ca.crt` for node authentication. + +### Using split CA certificates + +{{site.data.alerts.callout_danger}} +We do not recommend you use split CA certificates unless your organizational security practices mandate you to do so. +{{site.data.alerts.end}} + +You might encounter situations where you need separate CAs to sign and verify node and client certificates. In that case, you would need two CAs and their respective certificates and keys: `ca.crt` and `ca-client.crt`. + +**Node key and certificates** + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate to verify node certificates. +`ca-client.crt` | CA certificate to verify client certificates. +`node.crt` | Node certificate for when node acts as server.

All IP addresses and DNS names for the node must be listed in the `Subject Alternative Name`. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate).

Must be signed by the CA represented by `ca.crt`. +`node.key` | Server key corresponding to `node.crt`. +`client.node.crt` | Node certificate for when node acts as client. This certificate must be signed by the CA represented by `ca-client.crt`.

Must have `CN=node`. +`client.node.key` | Client key corresponding to `client.node.crt`. + +Optionally, if you have a certificate issued by a public CA to securely access the Admin UI, you need to place the certificate and key (`ui.crt` and `ui.key` respectively) in the directory specified by the `--certs-dir` flag. + +**Client key and certificates** + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`).

Must be signed by the CA represented by `ca-client.crt`. +`client..key` | Client key corresponding to `client..crt`. + +## Authentication for cloud storage + +See [Backup file URLs](backup.html#backup-file-urls) + +## Authentication best practice + +As a security best practice, we recommend that you rotate the node, client, or CA certificates in the following scenarios: + +- The node, client, or CA certificates are expiring soon. +- Your organization's compliance policy requires periodical certificate rotation. +- The key (for a node, client, or CA) is compromised. +- You need to modify the contents of a certificate, for example, to add another DNS name or the IP address of a load balancer through which a node can be reached. In this case, you would need to rotate only the node certificates. + +For details about when and how to change security certificates without restarting nodes, see [Rotate Security Certificates](rotate-certificates.html). + +## Background on public key cryptography and digital certificates + +As mentioned above, CockroachDB uses the [TLS 1.2](https://en.wikipedia.org/wiki/Transport_Layer_Security) security protocol that takes advantage of both symmetric (to encrypt data in flight) as well as asymmetric encryption (to establish a secure channel as well as **authenticate** the communicating parties). + +Authentication refers to the act of verifying the identity of the other party in communication. CockroachDB uses TLS 1.2 digital certificates for inter-node and client-node authentication, which require a Certificate Authority (CA) as well as keys and certificates for nodes, clients, and (optionally) the Admin UI. + +To understand how CockroachDB uses digital certificates, let's first understand what each of these terms means. + +Consider two people: Amy and Rosa, who want to communicate securely over an insecure computer network. The traditional solution is to use symmetric encryption that involves encrypting and decrypting a plaintext message using a shared key. Amy encrypts her message using the key and sends the encrypted message across the insecure channel. Rosa decrypts the message using the same key and reads the message. This seems like a logical solution until you realize that you need a secure communication channel to send the encryption key. + +To solve this problem, cryptographers came up with **asymmetric encryption** to set up a secure communication channel over which an encryption key can be shared. + +### Asymmetric encryption + +Asymmetric encryption involves a pair of keys instead of a single key. The two keys are called the **public key** and the **private key**. The keys consist of very long numbers linked mathematically in a way such that a message encrypted using a public key can only be decrypted using the private key and vice versa. The message cannot be decrypted using the same key that was used to encrypt the message. + +So going back to our example, Amy and Rosa both have their own public-private key pairs. They keep their private keys safe with themselves and publicly distribute their public keys. Now when Amy wants to send a message to Rosa, she requests Rosa's public key, encrypts the message using Rosa’s public key, and sends the encrypted message. Rosa uses her own private key to decrypt the message. + +But what if a malicious imposter intercepts the communication? The imposter might pose as Rosa and send their public key instead of Rosa’s. There's no way for Amy to know that the public key she received isn’t Rosa’s, so she would end up using the imposter's public key to encrypt the message and send it to the imposter. The imposter can use their own private key and decrypt and read the message, thus compromising the secure communication channel between Amy and Rosa. + +To prevent this security risk, Amy needs to be sure that the public key she received was indeed Rosa’s. That’s where the Certificate Authority (CA) comes into the picture. + +### Certificate authority + +Certificate authorities are established entities with their own public and private key pairs. They act as a root of trust and verify the identities of the communicating parties and validate their public keys. CAs can be public and paid entities (e.g., GeoTrust and Comodo), or public and free CAs (e.g., Let’s Encrypt), or your own organizational CA (e.g., CockroachDB CA). The CAs' public keys are typically widely distributed (e.g., your browser comes preloaded with certs from popular CAs like DigiCert, GeoTrust, and so on). + +Think of the CA as the passport authority of a country. When you want to get your passport as your identity proof, you submit an application to your country's passport authority. The application contains important identifying information about you: your name, address, nationality, date of birth, and so on. The passport authority verifies the information they received and validates your identity. They then issue a document - the passport - that can be presented anywhere in the world to verify your identity. For example, the TSA agent at the airport does not know you and has no reason to trust you are who you say you are. However, they trust the passport authority and thus accept your identity as presented on your passport because it has been verified and issued by the passport authority. + +Going back to our example and assuming that we trust the CA, Rosa needs to get her public key verified by the CA. She sends a CSR (Certificate Signing Request) to the CA that contains her public key and relevant identifying information. The CA will verify that it is indeed Rosa’s public key and information, _sign_ the CSR using the CA's own private key, and generate a digital document called the **digital certificate**. In our passport analogy, this is Rosa's passport containing verified identifying information about her and trusted by everyone who trusts the CA. The next time Rosa wants to establish her identity, she will present her digital certificate. + +### Digital certificate + +A public key is shared using a digital certificate signed by a CA using the CA's private key. The digital certificate contains: + +- The certificate owner’s public key +- Information about the certificate owner +- The CA's digital signature + +### Digital signature + +The CA's digital signature works as follows: The certificate contents are put through a mathematical function to create a **hash value**. This hash value is encrypted using the CA's private key to generate the **digital signature**. The digital signature is added to the digital certificate. In our example, the CA adds their digital signature to Rosa's certificate validating her identity and her public key. + +As discussed [earlier](#certificate-authority), the CA's public key is widely distributed. In our example, Amy already has the CA's public key. Now when Rosa presents her digital certificate containing her public key, Amy uses the CA's public key to decrypt the digital signature on Rosa's certificate and gets the hash value encoded in the digital signature. Amy also generates the hash value for the certificate on her own. If the hash values match, then Amy can be sure that the certificate and hence the public key it contains indeed belongs to Rosa; otherwise, she can determine that the communication channel has been compromised and refuse further contact. + +### How it all works together + +Let's see how the digital certificate is used in client-server communication: The client (e.g., a web browser) has the CA certificate (containing the CA's public key). When the client receives a server's certificate signed by the same CA, it can use the CA certificate to verify the server's certificate, thus validating the server's identity, and securely connect to the server. The important thing here is that the client needs to have the CA certificate. If you use your own organizational CA instead of a publicly established CA, you need to make sure you distribute the CA certificate to all the clients. + +## See also + +- [Client Connection Parameters](connection-parameters.html) +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Local Deployment](secure-a-cluster.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/authorization.md b/v20.2/authorization.md new file mode 100644 index 00000000000..7b32795976f --- /dev/null +++ b/v20.2/authorization.md @@ -0,0 +1,518 @@ +--- +title: Authorization +summary: Learn about the authorization features for secure CockroachDB clusters. +toc: true +redirect_from: [create-and-manage-users.html, roles.html] +--- + +User authorization is the act of defining access policies for authenticated CockroachDB users. CockroachDB allows you to create, manage, and remove your cluster's [users](#sql-users) and assign SQL-level [privileges](#assign-privileges) to the users. Additionally, you can use [role-based access management (RBAC)](#roles) for simplified user management. + +{{site.data.alerts.callout_info}} + Role-based access management (RBAC) is no longer an enterprise feature and is now freely available in the core version of CockroachDB. Also, for enhanced PostgreSQL compatibility, the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements. Note that even though the keywords are now interchangeable, it is still helpful to understand the distinction between the concepts (a "user" refers to an individual database user and a "role" refers to a group of database users). +{{site.data.alerts.end}} + +## SQL users + +A SQL user can interact with a CockroachDB database using the [built-in SQL shell](cockroach-sql.html) or through an application. + +### Create and manage users + +Use the [`CREATE USER`](create-user.html) and [`DROP USER`](drop-user.html) statements to create and remove users, the [`ALTER USER`](alter-user.html) statement to add or change a user's password and role options, the [`GRANT `](grant.html) and [`REVOKE `](revoke.html) statements to manage the user’s privileges, and the [`SHOW USERS`](show-users.html) statement to list users. + +A new user must be granted the required privileges for each database and table that the user needs to access. + +{{site.data.alerts.callout_info}} +By default, a new user belongs to the `public` role and has no privileges other than those assigned to the `public` role. For more information, see [Public role](#public-role). +{{site.data.alerts.end}} + +### `root` user + +The `root` user is created by default for each cluster. The `root` user is assigned to the [`admin` role](#admin-role) and has all privileges across the cluster. + +For secure clusters, in addition to [generating the client certificate](authentication.html#client-authentication) for the `root` user, you can assign or change the password for the `root` user using the [`ALTER USER`](alter-user.html) statement. + +## Roles + +{{site.data.alerts.callout_info}} + Role-based access management is no longer an enterprise feature and is now freely available in the core version of CockroachDB. +{{site.data.alerts.end}} + +A role is a group of users and/or other roles for which you can grant or revoke privileges as a whole. To simplify access management, create a role and grant privileges to the role, then create SQL users and grant them membership to the role. + +### Create and manage roles + +To create and manage your cluster's roles, use the following statements: + +Statement | Description +----------|------------ +[`CREATE ROLE`](create-role.html) | Create SQL roles. +[`DROP ROLE`](drop-role.html) | Remove one or more SQL roles. +[`GRANT `](grant-roles.html) | Add a role or user as a member to a role. +[`REVOKE `](revoke-roles.html) | Revoke a role or user's membership to a role. +[`GRANT `](grant.html) | Manage each role or user's SQL privileges for interacting with specific databases and tables. +[`REVOKE `](revoke.html) | Revoke privileges from users and/or roles. +[`SHOW ROLES`](show-roles.html) | List the roles for all databases. +[`SHOW GRANTS`](show-grants.html) | List the privileges granted to users. + +### Default roles + +The `admin` and `public` roles exist by default. + +#### `admin` role + +The `admin` role is created by default and cannot be dropped. Users belonging to the `admin` role have all privileges for all database objects across the cluster. The `root` user belongs to the `admin` role by default. + +An `admin` user is a member of the `admin` role. Only `admin` users can use [`CREATE ROLE`](create-role.html) and [`DROP ROLE`](drop-role.html). + +To assign a user to the `admin` role: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT admin TO ; +~~~ + +#### `public` role + +All new users and roles belong to the `public` role by default. You can grant and revoke the privileges on the `public` role. + +### Terminology + +#### Role admin + +A `role admin` is a member of the role that's allowed to grant or revoke role membership to other users for that specific role. To create a `role admin`, use [`WITH ADMIN OPTION`](grant-roles.html#grant-the-admin-option). + +{{site.data.alerts.callout_success}} +The terms “`admin` role” and “`role admin`” can be confusing. A user who is a member of the `admin` role has all privileges on all database objects across the entire cluster, whereas a `role admin` has privileges limited to the role they are a member of. Assign the `admin` role to a SQL user if you want the user to have privileges across the cluster. Make a SQL user the `role admin` if you want to limit the user’s privileges to its current role, but with an option to grant or revoke role membership to other users. This applies to the `admin` role as well - only admin users with the `WITH ADMIN OPTION` can add or remove other users from the `admin` role. +{{site.data.alerts.end}} + +#### Direct member + +A user or role that is an immediate member of the role. + +Example: A is a member of B. + +#### Indirect member + +A user or role that is a member of the role by association. + +Example: A is a member of C ... is a member of B where "..." is an arbitrary number of memberships. + +## Privileges + +When a user connects to a database, either via the built-in SQL client or a client driver, CockroachDB checks the user and role's privileges for each statement executed. If the user does not have sufficient privileges for a statement, CockroachDB gives an error. + +### Assign privileges + +Use the [`GRANT `](grant.html) and [`REVOKE `](revoke.html) statements to manage privileges for users and roles. + +Take the following points into consideration while granting privileges to roles and users: + +- When a role or user is granted privileges for a database, new tables created in the database will inherit the privileges, but the privileges can then be changed. To grant privileges to a user on all existing tables in a database, see [Grant privileges on all tables in a database](grant.html#grant-privileges-on-all-tables-in-a-database) + + {{site.data.alerts.callout_info}} + The user does not get privileges to existing tables in the database. + {{site.data.alerts.end}} + +- When a role or user is granted privileges for a table, the privileges are limited to the table. +- In CockroachDB, privileges are granted to users and roles at the database and table levels. They are not yet supported for other granularities such as columns or rows. +- The `root` user automatically belongs to the `admin` role and has the `ALL` privilege for new databases. +- For privileges required by specific statements, see the documentation for the respective [SQL statement](sql-statements.html). + +You can manage the following privileges for databases and tables: + +- `ALL` +- `CREATE` +- `DROP` +- `GRANT` +- `SELECT` +- `INSERT` +- `DELETE` +- `UPDATE` + +## Authorization best practices + +We recommend the following best practices to set up access control for your clusters: + +- Use the `root` user only for database administration tasks such as creating and managing other [users](#sql-users), creating and managing [roles](#roles), and creating and managing databases. Do not use the `root` user for applications; instead, create users or roles with specific [privileges](#assign-privileges) based on your application’s access requirements. +- Use the ["least privilege model"](https://en.wikipedia.org/wiki/Principle_of_least_privilege) to grant privileges to users and roles. + +## Example + +
+ + +
+ +
+ +The following example uses MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB [SQL statements](sql-statements.html). For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +Let's say we want to create the following access control setup for the `movr` database: + +- One database admin (named `db_admin`) who can perform all database operations for existing tables as well as for tables added in the future. +- One app user (named `app_user`) who can add, read update, and delete vehicles from the `vehicles` table. +- One user (named `report_user`) who can only read the `vehicles` table. + +1. Use the [`cockroach demo`](cockroach-demo.html) command to load the `movr` database and dataset into a CockroachDB cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach demo + ~~~ + +2. Create the database admin (named `db_admin`) who can perform all database operations for existing tables as well as for tables added in the future: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER db_admin; + ~~~ + +3. Grant all privileges on database `movr` to user `db_admin`: + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT ALL ON DATABASE movr TO db_admin; + ~~~ + +4. Grant all privileges on all tables in database `movr` to user `db_admin`: + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT ALL ON TABLE * TO db_admin; + ~~~ + +5. Verify that `db_admin` has all privileges: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS FOR db_admin; + ~~~ + + ~~~ + database_name | schema_name | table_name | grantee | privilege_type + +---------------+--------------------+----------------------------+----------+----------------+ + movr | crdb_internal | NULL | db_admin | ALL + movr | information_schema | NULL | db_admin | ALL + movr | pg_catalog | NULL | db_admin | ALL + movr | public | NULL | db_admin | ALL + movr | public | promo_codes | db_admin | ALL + movr | public | rides | db_admin | ALL + movr | public | user_promo_codes | db_admin | ALL + movr | public | users | db_admin | ALL + movr | public | vehicle_location_histories | db_admin | ALL + movr | public | vehicles | db_admin | ALL + (10 rows) + ~~~ + +6. As the `root` user, create a SQL user named `app_user` with permissions to add, read, update, and delete vehicles in the `vehicles` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER app_user; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT INSERT, DELETE, UPDATE, SELECT ON vehicles TO app_user; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS FOR app_user; + ~~~ + + ~~~ + database_name | schema_name | table_name | grantee | privilege_type + +---------------+-------------+------------+----------+----------------+ + movr | public | vehicles | app_user | DELETE + movr | public | vehicles | app_user | INSERT + movr | public | vehicles | app_user | SELECT + movr | public | vehicles | app_user | UPDATE + (4 rows) + ~~~ + +7. As the `root` user, create a SQL user named `report_user` with permissions to only read from the `vehicles` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT SELECT ON vehicles TO report_user; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS FOR report_user; + ~~~ + + ~~~ + database_name | schema_name | table_name | grantee | privilege_type + +---------------+-------------+------------+-------------+----------------+ + movr | public | vehicles | report_user | SELECT + (1 row) + ~~~ + +
+ +
+ +The following example uses MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). + +Let's say we want to create the following access control setup for the `movr` database: + +- Two database admins (named `db_admin_1` and `db_admin_2`) who can perform all database operations for existing tables as well as for tables added in the future. +- Three app users (named `app_user_1`, `app_user_2`, and `app_user_3`) who can add, read update, and delete vehicles from the `vehicles` table. +- Five users (named `report_user_1`, `report_user_2`, `report_user_3`, `report_user_4`, `report_user_5`) who can only read the `vehicles` table. + +1. Use the [`cockroach demo`](cockroach-demo.html) command to load the `movr` database and dataset into a CockroachDB cluster.: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach demo + ~~~ + +2. Create the database admin role (named `db_admin_role`) whose members can perform all database operations for existing tables as well as for tables added in the future: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE ROLE db_admin_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ROLES; + ~~~ + + ~~~ + username | options | member_of + ----------------+------------+------------ + admin | CREATEROLE | {} + db_admin_role | NOLOGIN | {} + root | CREATEROLE | {admin} + (3 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT ALL ON DATABASE movr TO db_admin_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT ALL ON TABLE * TO db_admin_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS ON DATABASE movr; + ~~~ + + ~~~ + database_name | schema_name | grantee | privilege_type + ----------------+--------------------+---------------+----------------- + movr | crdb_internal | admin | ALL + movr | crdb_internal | db_admin_role | ALL + movr | crdb_internal | root | ALL + movr | information_schema | admin | ALL + movr | information_schema | db_admin_role | ALL + movr | information_schema | root | ALL + movr | pg_catalog | admin | ALL + movr | pg_catalog | db_admin_role | ALL + movr | pg_catalog | root | ALL + movr | public | admin | ALL + movr | public | db_admin_role | ALL + movr | public | root | ALL + (12 rows) + ~~~ + +3. Create two database admin users (named `db_admin_1` and `db_admin_2`) and grant them membership to the `db_admin_role` role: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER db_admin_1; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER db_admin_2; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT db_admin_role TO db_admin_1, db_admin_2; + ~~~ + +4. Create a role named `app_user_role` whose members can add, read update, and delete vehicles to the `vehicles` table. + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE ROLE app_user_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ROLES; + ~~~ + + ~~~ + username | options | member_of + ----------------+------------+------------------ + admin | CREATEROLE | {} + app_user_role | NOLOGIN | {} + db_admin_1 | | {db_admin_role} + db_admin_2 | | {db_admin_role} + db_admin_role | NOLOGIN | {} + root | CREATEROLE | {admin} + (6 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT INSERT, UPDATE, DELETE, SELECT ON TABLE vehicles TO app_user_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS ON vehicles; + ~~~ + + ~~~ + database_name | schema_name | table_name | grantee | privilege_type + ----------------+-------------+------------+---------------+----------------- + movr | public | vehicles | admin | ALL + movr | public | vehicles | app_user_role | DELETE + movr | public | vehicles | app_user_role | INSERT + movr | public | vehicles | app_user_role | SELECT + movr | public | vehicles | app_user_role | UPDATE + movr | public | vehicles | db_admin_role | ALL + movr | public | vehicles | root | ALL + (7 rows) + ~~~ + +5. Create three app users (named `app_user_1`, `app_user_2`, and `app_user_3`) and grant them membership to the `app_user_role` role: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER app_user_1; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER app_user_2; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER app_user_3; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT app_user_role TO app_user_1, app_user_2, app_user_3; + ~~~ + +6. Create a role named `report_user_role` whose members can only read the `vehicles` table. + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE ROLE report_user_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ROLES; + ~~~ + + ~~~ + username | options | member_of + -------------------+------------+------------------ + admin | CREATEROLE | {} + app_user_1 | | {app_user_role} + app_user_2 | | {app_user_role} + app_user_3 | | {app_user_role} + app_user_role | NOLOGIN | {} + db_admin_1 | | {db_admin_role} + db_admin_2 | | {db_admin_role} + db_admin_role | NOLOGIN | {} + report_user_role | NOLOGIN | {} + root | CREATEROLE | {admin} + (10 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT SELECT ON vehicles TO report_user_role; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW GRANTS ON vehicles; + ~~~ + + ~~~ + database_name | schema_name | table_name | grantee | privilege_type + ----------------+-------------+------------+------------------+----------------- + movr | public | vehicles | admin | ALL + movr | public | vehicles | app_user_role | DELETE + movr | public | vehicles | app_user_role | INSERT + movr | public | vehicles | app_user_role | SELECT + movr | public | vehicles | app_user_role | UPDATE + movr | public | vehicles | db_admin_role | ALL + movr | public | vehicles | report_user_role | SELECT + movr | public | vehicles | root | ALL + (8 rows) + ~~~ + +7. Create five report users (named `report_user_1`, `report_user_2`, `report_user_3`, `report_user_4`, and `report_user_5`) and grant them membership to the `report_user_role` role: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user_1; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user_2; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user_3; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user_4; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER report_user_5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT report_user_role TO report_user_1, report_user_2, report_user_3, report_user_4, report_user_5; + ~~~ + +
+ +## See also + +- [Client Connection Parameters](connection-parameters.html) +- [SQL Statements](sql-statements.html) +- [`CREATE USER`](create-user.html) +- [`ALTER USER`](alter-user.html) +- [`DROP USER`](drop-user.html) +- [`SHOW USERS`](show-users.html) +- [`CREATE ROLE`](create-role.html) +- [`DROP ROLE`](drop-role.html) +- [`SHOW ROLES`](show-roles.html) +- [`GRANT `](grant.html) +- [`GRANT `](grant-roles.html) +- [`REVOKE `](revoke.html) +- [`REVOKE `](revoke-roles.html) +- [`SHOW GRANTS`](show-grants.html) diff --git a/v20.2/backup-and-restore-advanced-options.md b/v20.2/backup-and-restore-advanced-options.md new file mode 100644 index 00000000000..69f7a2460a6 --- /dev/null +++ b/v20.2/backup-and-restore-advanced-options.md @@ -0,0 +1,392 @@ +--- +title: Back up and Restore Data - Advanced Options +summary: Learn about the advanced options you can use when you backup and restore a CockroachDB cluster. +toc: true +--- + + The ability to [backup a full cluster](backup.html#backup-a-cluster) has been added and the syntax for [incremental backups](backup.html#create-incremental-backups) is simplified. Because of these two changes, [basic backup usage](backup-and-restore.html) is now sufficient for most CockroachDB clusters. However, you may want to control your backup and restore options more explicitly, which now falls under advanced usage. + +This doc provides information about the advanced options you can use when you [backup](backup.html) and [restore](restore.html) data in a CockroachDB cluster: + +- [Incremental backups with a specified destination](#incremental-backups-with-explicitly-specified-destinations) +- [Backup with revision history and point-in-time restore](#backup-with-revision-history-and-point-in-time-restore) +- [Locality-aware backup and restore](#locality-aware-backup-and-restore) +- [Encrypted backup and restore](#encrypted-backup-and-restore) +- [Restore into a different database](#restore-into-a-different-database) +- [Remove the foreign key before restore](#remove-the-foreign-key-before-restore) +- [Restoring users from `system.users` backup](#restoring-users-from-system-users-backup) + +{{site.data.alerts.callout_info}} +The advanced options covered in this doc are included in [`BACKUP`](backup.html), which is an [enterprise-only](https://www.cockroachlabs.com/product/cockroachdb/) feature. For non-enterprise backups, see [`cockroach dump`](cockroach-dump.html). +{{site.data.alerts.end}} + +## Incremental backups with explicitly specified destinations + +To explicitly control where your incremental backups go, use the [`INCREMENTAL FROM`](backup.html#synopsis) syntax: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP DATABASE bank \ +TO 'gs://acme-co-backup/db/bank/2017-03-29-nightly' \ +AS OF SYSTEM TIME '-10s' \ +INCREMENTAL FROM 'gs://acme-co-backup/database-bank-2017-03-27-weekly', 'gs://acme-co-backup/database-bank-2017-03-28-nightly' WITH revision_history; +~~~ + +For an example of the simplified incremental backup syntax, see [Create incremental backups](backup.html#create-incremental-backups). + +## Backup with revision history and point-in-time restore + +You can create full or incremental backups [with revision history](backup.html#with-revision-history): + +- Taking full backups with revision history allows you to back up every change made within the garbage collection period leading up to and including the given timestamp. +- Taking incremental backups with revision history allows you to back up every change made since the last backup and within the garbage collection period leading up to and including the given timestamp. You can take incremental backups with revision history even when your previous full or incremental backups were taken without revision history. + +You can configure garbage collection periods using the `ttlseconds` [replication zone setting](configure-replication-zones.html). Taking backups with revision history allows for point-in-time restores within the revision history. + +### Create a backup with revision history + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +'gs://acme-co-backup/test-cluster-2017-03-27-weekly' \ +AS OF SYSTEM TIME '-10s' WITH revision_history; +~~~ + +### Point-in-time restore + +If the full or incremental backup was taken [with revision history](#backup-with-revision-history-and-point-in-time-restore), you can restore the data as it existed at an arbitrary point-in-time within the revision history captured by that backup. Use the [`AS OF SYSTEM TIME`](as-of-system-time.html) clause to specify the point-in-time. + + Additionally, if you want to restore a specific incremental backup, you can do so by specifying the `end_time` of the backup by using the [`AS OF SYSTEM TIME`](as-of-system-time.html) clause. To find the incremental backup's `end_time`, use [`SHOW BACKUP`](show-backup.html). + +If you do not specify a point-in-time, the data will be restored to the backup timestamp; that is, the restore will work as if the data was backed up without revision history. + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +AS OF SYSTEM TIME '2017-02-26 10:00:00'; +~~~ + +### Point-in-time restore from incremental backups + +Restoring from incremental backups requires previous full and incremental backups. In this example, `-weekly` is the full backup and the two `-nightly` are incremental backups: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM \ +'gs://acme-co-backup/database-bank-2017-03-27-weekly', 'gs://acme-co-backup/database-bank-2017-03-28-nightly', 'gs://acme-co-backup/database-bank-2017-03-29-nightly' \ +AS OF SYSTEM TIME '2017-02-28 10:00:00'; +~~~ + +## Locality-aware backup and restore + +You can create locality-aware backups such that each node writes files only to the backup destination that matches the [node locality](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes) configured at [node startup](cockroach-start.html). + +This is useful for: + +- Reducing cloud storage data transfer costs by keeping data within cloud regions. +- Helping you comply with data domiciling requirements. + +A locality-aware backup is specified by a list of URIs, each of which has a `COCKROACH_LOCALITY` URL parameter whose single value is either `default` or a single locality key-value pair such as `region=us-east`. At least one `COCKROACH_LOCALITY` must be the `default`. Given a list of URIs that together contain the locations of all of the files for a single locality-aware backup, [`RESTORE` can read in that backup](#restore-from-a-locality-aware-backup). + +{{site.data.alerts.callout_info}} +The locality query string parameters must be [URL-encoded](https://en.wikipedia.org/wiki/Percent-encoding). +{{site.data.alerts.end}} + +During locality-aware backups, backup file placement is determined by leaseholder placement, as each node is responsible for backing up the ranges for which it is the leaseholder. Nodes write files to the backup storage location whose locality matches their own node localities, with a preference for more specific values in the locality hierarchy. If there is no match, the `default` locality is used. + +### Create a locality-aware backup + +For example, to create a locality-aware backup where nodes with the locality `region=us-west` write backup files to `s3://us-west-bucket`, and all other nodes write to `s3://us-east-bucket` by default, run: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO + ('s3://us-east-bucket?COCKROACH_LOCALITY=default', 's3://us-west-bucket?COCKROACH_LOCALITY=region%3Dus-west'); +~~~ + +can be restored by running: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM ('s3://us-east-bucket', 's3://us-west-bucket'); +~~~ + +Note that the first URI in the list has to be the URI specified as the `default` URI when the backup was created. If you have moved your backups to a different location since the backup was originally taken, the first URI must be the new location of the files originally written to the `default` location. + +### Restore from a locality-aware backup + +You can create locality-aware backups such that each node writes files only to the backup destination that matches the [node locality](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes) configured at [node startup](cockroach-start.html). + +Given a list of URIs that together contain the locations of all of the files for a single [locality-aware backup](#create-a-locality-aware-backup), [`RESTORE`][restore] can read in that backup. Note that the list of URIs passed to [`RESTORE`][restore] may be different from the URIs originally passed to [`BACKUP`][backup]. This is because it's possible to move the contents of one of the parts of a locality-aware backup (i.e., the files written to that destination) to a different location, or even to consolidate all the files for a locality-aware backup into a single location. + +When restoring a [full backup](backup-and-restore.html#full-backups), the cluster data is restored first, then the system table data "as is." This means that the restored zone configurations can point to regions that do not have active nodes in the new cluster. For example, if your full backup has the following [zone configurations](configure-zone.html): + +~~~ sql +> ALTER PARTITION europe_west OF INDEX movr.public.rides@primary \ + CONFIGURE ZONE USING constraints = '[+region=europe-west1]'; + +> ALTER PARTITION us_east OF INDEX movr.public.rides@primary \ + CONFIGURE ZONE USING constraints = '[+region=us-east1]'; + +> ALTER PARTITION us_west OF INDEX movr.public.rides@primary \ + CONFIGURE ZONE USING constraints = '[+region=us-west1]'; +~~~ + +And the restored cluster does not have [nodes with the locality](partitioning.html#node-attributes) `region=us-west1`, the restored cluster will still have a zone configuration for `us-west1`. This means that the cluster's data will _not_ be reshuffled to `us-west1` because the region does not exist. The data will be distributed as if the zone configuration does not exist. For the data to be distributed correctly, you can [add node(s)](cockroach-start.html) with the missing region or [remove the zone configuration](configure-zone.html#remove-a-replication-zone). + + +{{site.data.alerts.callout_info}} +[`RESTORE`][restore] is not truly locality-aware; while restoring from backups, a node may read from a store that does not match its locality. This can happen in the cases that either the [`BACKUP`][backup] or [`RESTORE`][restore] was not full cluster. Note that during a locality-aware restore, some data may be temporarily located on another node before it is eventually relocated to the appropriate node. To avoid this, you can [manually restore zone configurations from a locality-aware backup](#manually-restore-zone-configurations-from-a-locality-aware-backup). +{{site.data.alerts.end}} + +### Create an incremental locality-aware backup + +To create an incremental locality-aware backup from a full locality-aware backup, the syntax the same as it is for [regular incremental backups](backup.html#create-incremental-backups). If you backup to a destination already containing a full backup, an incremental backup will be appended to the full backup in a subdirector. For example: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO + ('s3://us-east-bucket?COCKROACH_LOCALITY=default', 's3://us-west-bucket?COCKROACH_LOCALITY=region%3Dus-west'); +~~~ + +{{site.data.alerts.callout_info}} +It is recommend that the same localities be included for every incremental backup in the series of backups; however, only the `default` locality is required. When [restoring from an incremental locality-aware backup](#restore-from-an-incremental-locality-aware-backup), you need to include _every_ locality ever used, even if it was only used once. +{{site.data.alerts.end}} + +And if you want to explicitly control where your incremental backups go, use the `INCREMENTAL FROM` syntax: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO (${uri_1}, ${uri_2}, ...) INCREMENTAL FROM ${full_backup_uri} ...; +~~~ + +For example, to create an incremental locality-aware backup from a previous full locality-aware backup where nodes with the locality `region=us-west` write backup files to `s3://us-west-bucket`, and all other nodes write to `s3://us-east-bucket` by default, run: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +('s3://us-east-bucket/test-cluster-2019-10-08-nightly?COCKROACH_LOCALITY=default', 's3://us-west-bucket/test-cluster-2019-10-08-nightly?COCKROACH_LOCALITY=region%3Dus-west') +INCREMENTAL FROM 's3://us-east-bucket/test-cluster-2019-10-07-weekly'; +~~~ + +{{site.data.alerts.callout_info}} +Note that only the backup URIs you set as the `default` when you created the previous backup(s) are needed in the `INCREMENTAL FROM` clause of your incremental `BACKUP` statement (as shown in the example). This is because the `default` destination for a locality-aware backup contains a manifest file that contains all the metadata required to create additional incremental backups based on it. +{{site.data.alerts.end}} + +### Restore from an incremental locality-aware backup + +A locality-aware backup URI can also be used in place of any incremental backup URI in [`RESTORE`][restore]. + +For example, an incremental locality-aware backup created with + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO + ('s3://us-east-bucket/database-bank-2019-10-08-nightly?COCKROACH_LOCALITY=default', 's3://us-west-bucket/database-bank-2019-10-08-nightly?COCKROACH_LOCALITY=region%3Dus-west') + INCREMENTAL FROM + 's3://us-east-bucket/database-bank-2019-10-07-weekly'; +~~~ + +can be restored by running: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM + ('s3://us-east-bucket/database-bank-2019-10-07-weekly', 's3://us-west-bucket/database-bank-2019-10-07-weekly'), + ('s3://us-east-bucket/database-bank-2019-10-08-nightly', 's3://us-west-bucket/database-bank-2019-10-08-nightly'); +~~~ + +{{site.data.alerts.callout_info}} +When restoring from an incremental locality-aware backup, you need to include _every_ locality ever used, even if it was only used once. +{{site.data.alerts.end}} + +### Create an incremental locality-aware backup from a previous locality-aware backup + +To make an incremental locality-aware backup from another locality-aware backup, the syntax is as follows: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO ({uri_1}, {uri_2}, ...) INCREMENTAL FROM {full_backup}, {incr_backup_1}, {incr_backup_2}, ...; +~~~ + +For example, let's say you normally run a full backup every Monday, followed by incremental backups on the remaining days of the week. + +By default, all nodes send their backups to your `s3://us-east-bucket`, except for nodes in `region=us-west`, which will send their backups to `s3://us-west-bucket`. + +If today is Thursday, October 10th, 2019, your `BACKUP` statement will list the following backup URIs: + +- The full locality-aware backup URI from Monday, e.g., + - `s3://us-east-bucket/test-cluster-2019-10-07-weekly` +- The incremental backup URIs from Tuesday and Wednesday, e.g., + - `s3://us-east-bucket/test-cluster-2019-10-08-nightly` + - `s3://us-east-bucket/test-cluster-2019-10-09-nightly` + +Given the above, to take the incremental locality-aware backup scheduled for today (Thursday), you will run: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO + ('s3://us-east-bucket/test-cluster-2019-10-10-nightly?COCKROACH_LOCALITY=default', 's3://us-west-bucket/test-cluster-2019-10-10-nightly?COCKROACH_LOCALITY=region%3Dus-west') + INCREMENTAL FROM + 's3://us-east-bucket/test-cluster-2019-10-07-weekly', + 's3://us-east-bucket/test-cluster-2019-10-08-nightly', + 's3://us-east-bucket/test-cluster-2019-10-09-nightly'; +~~~ + +{{site.data.alerts.callout_info}} +Note that only the backup URIs you set as the `default` when you created the previous backup(s) are needed in the `INCREMENTAL FROM` clause of your incremental `BACKUP` statement (as shown in the example). This is because the `default` destination for a locality-aware backup contains a manifest file that contains all the metadata required to create additional incremental backups based on it. +{{site.data.alerts.end}} + +### Manually restore zone configurations from a locality-aware backup + +During a [locality-aware restore](#restore-from-a-locality-aware-backup), some data may be temporarily located on another node before it is eventually relocated to the appropriate node. To avoid this, you need to manually restore [zone configurations](configure-replication-zones.html) first: + +Once the locality-aware restore has started, [pause the restore](pause-job.html): + +{% include copy-clipboard.html %} +~~~ sql +> PAUSE JOB 27536791415282; +~~~ + +The `system.zones` table stores your cluster's [zone configurations](configure-replication-zones.html), which will prevent the data from rebalancing. To restore them, you must restore the `system.zones` table into a new database because you cannot drop the existing `system.zones` table: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE system.zones \ +FROM 'azure://acme-co-backup?AZURE_ACCOUNT_KEY=hash&AZURE_ACCOUNT_NAME=acme-co' \ +WITH into_db = 'newdb'; +~~~ + +After it's restored into a new database, you can write the restored `zones` table data to the cluster's existing `system.zones` table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO system.zones SELECT * FROM newdb.zones; +~~~ + +Then drop the temporary table you created: + +{% include copy-clipboard.html %} +~~~ sql +> DROP TABLE newdb.zones; +~~~ + +Then, [resume the restore](resume-job.html): + +{% include copy-clipboard.html %} +~~~ sql +> RESUME JOB 27536791415282; +~~~ + +## Encrypted backup and restore + +{% include {{ page.version.version }}/backups/encrypted-backup-description.md %} + +### Create an encrypted backup + + To create an [encrypted backup](#encrypted-backup-and-restore), use the `encryption_passphrase` option: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +'gs://acme-co-backup/test-cluster' \ +WITH encryption_passphrase = 'password123'; +~~~ +~~~ + job_id | status | fraction_completed | rows | index_entries | bytes +---------------------+-----------+--------------------+------+---------------+--------- + 543214409874014209 | succeeded | 1 | 2597 | 1028 | 467701 +(1 row) +~~~ + +To [restore](restore.html), use the same `encryption_passphrase`: + +### Restore from an encrypted backup + + To decrypt an [encrypted backup](#encrypted-backup-and-restore), use the `encryption_passphrase` option and the same passphrase that was used to create the backup. + +For example, the encrypted backup created in the [previous example](#create-an-encrypted-backup): + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +'gs://acme-co-backup/test-cluster' \ +WITH encryption_passphrase = 'password123'; +~~~ + +Can be restored with: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM \ +'gs://acme-co-backup/test-cluster' \ +WITH encryption_passphrase = 'password123'; +~~~ +~~~ + job_id | status | fraction_completed | rows | index_entries | bytes +---------------------+-----------+--------------------+------+---------------+--------- + 543217488273801217 | succeeded | 1 | 2597 | 1028 | 467701 +(1 row) +~~~ + +## Other restore usages + +### Restore into a different database + +By default, tables and views are restored to the database they originally belonged to. However, using the [`into_db`](restore.html#into_db) option, you can control the target database. + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE bank.customers \ +FROM 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +WITH into_db = 'newdb'; +~~~ + +### Remove the foreign key before restore + +By default, tables with [Foreign Key](foreign-key.html) constraints must be restored at the same time as the tables they reference. However, using the [`skip_missing_foreign_keys`](restore.html#skip_missing_foreign_keys) option you can remove the Foreign Key constraint from the table and then restore it. + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE bank.accounts \ +FROM 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +WITH skip_missing_foreign_keys; +~~~ + +### Restoring users from `system.users` backup + +The `system.users` table stores your cluster's usernames and their hashed passwords. To restore them, you must restore the `system.users` table into a new database because you cannot drop the existing `system.users` table. + +After it's restored into a new database, you can write the restored `users` table data to the cluster's existing `system.users` table. + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE system.users \ +FROM 'azure://acme-co-backup/table-users-2017-03-27-full?AZURE_ACCOUNT_KEY=hash&AZURE_ACCOUNT_NAME=acme-co' \ +WITH into_db = 'newdb'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO system.users SELECT * FROM newdb.users; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP TABLE newdb.users; +~~~ + +## See also + +- [Backup and Restore Data](backup-and-restore.html) +- [`BACKUP`][backup] +- [`RESTORE`][restore] +- [`SQL DUMP`](cockroach-dump.html) +- [`IMPORT`](import-data.html) +- [Use the Built-in SQL Client](cockroach-sql.html) +- [Other Cockroach Commands](cockroach-commands.html) + + + +[backup]: backup.html +[restore]: restore.html diff --git a/v20.2/backup-and-restore.md b/v20.2/backup-and-restore.md new file mode 100644 index 00000000000..201e17079be --- /dev/null +++ b/v20.2/backup-and-restore.md @@ -0,0 +1,225 @@ +--- +title: Back up and Restore Data +summary: Learn how to back up and restore a CockroachDB cluster. +toc: true +redirect_from: +- back-up-data.html +- restore-data.html +--- + +Because CockroachDB is designed with high fault tolerance, backups are primarily needed for disaster recovery (i.e., if your cluster loses a majority of its nodes). Isolated issues (such as small-scale node outages) do not require any intervention. However, as an operational best practice, **we recommend taking regular backups of your data**. + +Based on your [license type](https://www.cockroachlabs.com/pricing/), CockroachDB offers two methods to backup and restore your cluster's data: [Enterprise](#perform-enterprise-backup-and-restore) and [Core](#perform-core-backup-and-restore). + +## Perform Enterprise backup and restore + +If you have an [Enterprise license](enterprise-licensing.html), you can use the [`BACKUP`][backup] statement to efficiently back up your cluster's schemas and data to popular cloud services such as AWS S3, Google Cloud Storage, or NFS, and the [`RESTORE`][restore] statement to efficiently restore schema and data as necessary. + +{{site.data.alerts.callout_success}} +We recommend [automating daily backups of your cluster](#automated-full-and-incremental-backups). To automate backups, you must have a client send the `BACKUP` statement to the cluster. Once the backup is complete, your client will receive a `BACKUP` response. +{{site.data.alerts.end}} + +### Full backups + +In most cases, **it's recommended to take full nightly backups of your cluster**. A full cluster backup allows you to do the following: + +- Restore table(s) from the cluster +- Restore database(s) from the cluster +- Restore a full cluster + +To do a full cluster backup, use the [`BACKUP`](backup.html) statement: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO ''; +~~~ + +If it's ever necessary, you can use the [`RESTORE`][restore] statement to restore a table: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE TABLE bank.customers FROM ''; +~~~ + +Or to restore a database: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE DATABASE bank FROM ''; +~~~ + +Or to restore your full cluster: + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM ''; +~~~ + +{{site.data.alerts.callout_info}} +A full cluster restore can only be run on a target cluster that has _never_ had user-created databases or tables. +{{site.data.alerts.end}} + +### Full and incremental backups + +If your cluster grows too large for nightly full backups, you can take less frequent full backups (e.g., weekly) with nightly incremental backups. Incremental backups are storage efficient and faster than full backups for larger clusters. + +Periodically run the [`BACKUP`][backup] command to take a full backup of your database: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO ''; +~~~ + +Then, create nightly incremental backups based off of the full backups you've already created. If you backup to a destination already containing a full backup, an incremental backup will be appended to the full backup in a subdirectory: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO ''; +~~~ + +{{site.data.alerts.callout_info}} +For an example on how to specify the destination of an incremental backup, see [Backup and Restore - Advanced Options](backup-and-restore-advanced-options.html#incremental-backups-with-explicitly-specified-destinations) +{{site.data.alerts.end}} + +If it's ever necessary, you can then use the [`RESTORE`][restore] command to restore your cluster, database(s), and/or table(s). [Restoring from incremental backups](restore.html#restore-from-incremental-backups) requires previous full and incremental backups. To restore from a destination containing the full backup, as well as the automatically appended incremental backups (that are stored as subdirectories, like in the example above): + +{% include copy-clipboard.html %} +~~~ sql +> RESTORE FROM ''; +~~~ + +### Examples + +#### Automated full and incremental backups + +You can automate your backups using scripts and your preferred method of automation, such as cron jobs. + +For your reference, we have created this [sample backup script](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/backup.sh) that you can customize to automate your backups. + +In the sample script, configure the day of the week for which you want to create full backups. Running the script daily will create a full backup on the configured day, and on other days, it'll create incremental backups. The script tracks the recently created backups in a separate file titled `backup.txt` and uses this file as a base for the subsequent incremental backups. + +1. Download the [sample backup script](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/backup.sh): + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/prod-deployment/backup.sh + ~~~ + + Alternatively, you can create the file yourself and copy the script into it: + + {% include copy-clipboard.html %} + ~~~ shell + #!/bin/bash + + set -euo pipefail + + # This script creates full backups when run on the configured + # day of the week and incremental backups when run on other days, and tracks + # recently created backups in a file to pass as the base for incremental backups. + + full_day="" # Must match (including case) the output of `LC_ALL=C date +%A`. + what="" # Leave empty for full cluster backup, or add "DATABASE database_name" to backup a database. + base="/backups" # The URL where you want to store the backup. + extra="" # Any additional parameters that need to be appended to the BACKUP URI (e.g., AWS key params). + recent=recent_backups.txt # File in which recent backups are tracked. + backup_parameters= # e.g., "WITH revision_history" + + # Customize the `cockroach sql` command with `--host`, `--certs-dir` or `--insecure`, and additional flags as needed to connect to the SQL client. + runsql() { cockroach sql --insecure -e "$1"; } + + destination="${base}/$(date +"%Y%m%d-%H%M")${extra}" + + prev= + while read -r line; do + [[ "$prev" ]] && prev+=", " + prev+="'$line'" + done < "$recent" + + if [[ "$(LC_ALL=C date +%A)" = "$full_day" || ! "$prev" ]]; then + runsql "BACKUP $what TO '$destination' AS OF SYSTEM TIME '-1m' $backup_parameters" + echo "$destination" > "$recent" + else + destination="${base}/$(date +"%Y%m%d-%H%M")-inc${extra}" + runsql "BACKUP $what TO '$destination' AS OF SYSTEM TIME '-1m' INCREMENTAL FROM $prev $backup_parameters" + echo "$destination" >> "$recent" + fi + + echo "backed up to ${destination}" + ~~~ + +2. In the sample backup script, customize the values for the following variables: + + Variable | Description + -----|------------ + `full_day` | The day of the week on which you want to take a full backup. + `what` | Leave empty for a full cluster backup. Otherwise, add `DATABASE ` to back up a database (i.e., create backups of all tables and views in the database). + `base` | The URL where you want to store the backup.

URL format: `[scheme]://[host]/[path]`

For information about the components of the URL, see [Backup File URLs](backup.html#backup-file-urls). + `extra`| The parameters required for the storage.

Parameters format: `?[parameters]`

For information about the storage parameters, see [Backup File URLs](backup.html#backup-file-urls). + `backup_parameters` | Additional [backup parameters](backup.html#parameters) you might want to specify. + + Also customize the `cockroach sql` command with `--host`, `--certs-dir` or `--insecure`, and [additional flags](cockroach-sql.html#flags) as required. + +3. Change the file permissions to make the script executable: + + {% include copy-clipboard.html %} + ~~~ shell + $ chmod +x backup.sh + ~~~ + +4. Run the backup script: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./backup.sh + ~~~ + +{{site.data.alerts.callout_info}} +If you miss an incremental backup, delete the `recent_backups.txt` file and run the script. It'll take a full backup for that day and incremental backups for subsequent days. +{{site.data.alerts.end}} + +#### Advanced examples + +{% include {{ page.version.version }}/backups/advanced-examples-list.md %} + +## Perform Core backup and restore + +If you do not have an Enterprise license, you can perform a core backup. Run the [`cockroach dump`](cockroach-dump.html) command to dump all the tables in the database to a new file (e.g., `backup.sql`): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump > backup.sql +~~~ + +To restore a database from a core backup, use the [`IMPORT PGDUMP`](import.html#import-a-cockroachdb-dump-file) statement: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --execute="IMPORT PGDUMP 's3://your-external-storage/backup.sql?AWS_ACCESS_KEY_ID=[placeholder]&AWS_SECRET_ACCESS_KEY=[placeholder]'" \ + +~~~ + +You can also [use the `cockroach sql` command](cockroach-dump.html#restore-a-table-from-a-backup-file) to execute the [`CREATE TABLE`](create-table.html) and [`INSERT`](insert.html) statements in the backup file: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --database=[database name] < backup.sql +~~~ + +{{site.data.alerts.callout_success}} +If you created a backup from another database and want to import it into CockroachDB, see the [Migration Overview](migration-overview.html). +{{site.data.alerts.end}} + +## See also + +- [Back up and Restore Data - Advanced Options](backup-and-restore-advanced-options.html) +- [`BACKUP`][backup] +- [`RESTORE`][restore] +- [`SQL DUMP`](cockroach-dump.html) +- [`IMPORT`](import-data.html) +- [Use the Built-in SQL Client](cockroach-sql.html) +- [Other Cockroach Commands](cockroach-commands.html) + + + +[backup]: backup.html +[restore]: restore.html diff --git a/v20.2/backup.md b/v20.2/backup.md new file mode 100644 index 00000000000..1bf79ea6831 --- /dev/null +++ b/v20.2/backup.md @@ -0,0 +1,228 @@ +--- +title: BACKUP +summary: Back up your CockroachDB cluster to a cloud storage services such as AWS S3, Google Cloud Storage, or other NFS. +toc: true +--- + +{{site.data.alerts.callout_info}} +`BACKUP` is an [enterprise-only](https://www.cockroachlabs.com/product/cockroachdb/) feature. For non-enterprise backups, see [`cockroach dump`](cockroach-dump.html). +{{site.data.alerts.end}} + +CockroachDB's `BACKUP` [statement](sql-statements.html) allows you to create [full or incremental backups](backup-and-restore.html#perform-enterprise-backup-and-restore) of your cluster's schema and data that are consistent as of a given timestamp. + + You can [backup a full cluster](#backup-a-cluster), which includes: + +- All user tables +- Relevant system tables +- All [databases](create-database.html) +- All [tables](create-table.html) (which automatically includes their [indexes](indexes.html)) +- All [views](views.html) + +You can also back up: + +- [An individual database](#backup-a-database), which includes all of its tables and views +- [An individual table](#backup-a-table-or-view), which includes its indexes and views + +Because CockroachDB is designed with high fault tolerance, these backups are designed primarily for disaster recovery (i.e., if your cluster loses a majority of its nodes) through [`RESTORE`](restore.html). Isolated issues (such as small-scale node outages) do not require any intervention. + +{{site.data.alerts.callout_info}} +`BACKUP` only offers table-level granularity; it _does not_ support backing up subsets of a table. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_success}} +To view the contents of an enterprise backup created with the `BACKUP` statement, use [`SHOW BACKUP`](show-backup.html). +{{site.data.alerts.end}} + +## Required privileges + +Only members of the `admin` role can run `BACKUP`. By default, the `root` user belongs to the `admin` role. + +## Synopsis + +
+{% include {{ page.version.version }}/sql/diagrams/backup.html %} +
+ +## Parameters + + Parameter | Description +------------|------------- +`table_pattern` | The table(s) or [view(s)](views.html) you want to back up. +`database_name` | The name of the database(s) you want to back up (i.e., create backups of all tables and views in the database).| +`destination` | The URL where you want to store the backup.

For information about this URL structure, see [Backup File URLs](#backup-file-urls). +`timestamp` | Back up data as it existed as of [`timestamp`](as-of-system-time.html). The `timestamp` must be more recent than your cluster's last garbage collection (which defaults to occur every 25 hours, but is [configurable per table](configure-replication-zones.html#replication-zone-variables)). +`full_backup_location` | Create an incremental backup using the full backup stored at the URL `full_backup_location` as its base. For information about this URL structure, see [Backup File URLs](#backup-file-urls).

**Note:** It is not possible to create an incremental backup if one or more tables were [created](create-table.html), [dropped](drop-table.html), or [truncated](truncate.html) after the full backup. In this case, you must create a new [full backup](#full-backups). +`incremental_backup_location` | Create an incremental backup that includes all backups listed at the provided URLs.

Lists of incremental backups must be sorted from oldest to newest. The newest incremental backup's timestamp must be within the table's garbage collection period.

For information about this URL structure, see [Backup File URLs](#backup-file-urls).

For more information about garbage collection, see [Configure Replication Zones](configure-replication-zones.html#replication-zone-variables). +`kv_option_list` | Control the backup behavior with a comma-separated list of [these options](#options). + +{{site.data.alerts.callout_info}} +The `BACKUP` statement cannot be used within a [transaction](transactions.html). +{{site.data.alerts.end}} + +### Options + + Option | Value | Description +-----------------------------------------------------------------+-------------------------+------------------------------ +`revision_history` | N/A | Create a backup with full [revision history](backup-and-restore-advanced-options.html#backup-with-revision-history-and-point-in-time-restore), which records every change made to the cluster within the garbage collection period leading up to and including the given timestamp. +`encryption_passphrase` | [`STRING`](string.html) | The passphrase used to [encrypt the files](backup-and-restore-advanced-options.html#encrypted-backup-and-restore) (`BACKUP` manifest and data files) that the `BACKUP` statement generates. This same passphrase is needed to decrypt the file when it is used to [restore](backup-and-restore-advanced-options.html#restore-from-an-encrypted-backup) and to list the contents of the backup when using [`SHOW BACKUP`](show-backup.html). + +For more information about these options, see [Back up and Restore Data - Advanced Options](backup-and-restore-advanced-options.html). + +### Backup file URLs + +CockroachDB uses the URL provided to construct a secure API call to the service you specify. The path to each backup must be unique, and the URL for your backup's destination/locations must use the following format: + +{% include {{ page.version.version }}/misc/external-urls.md %} + +## Functional details + +### Object dependencies + +Dependent objects must be backed up at the same time as the objects they depend on. + +Object | Depends On +-------|----------- +Table with [foreign key](foreign-key.html) constraints | The table it `REFERENCES`; however, this dependency can be [removed during the restore](restore.html#skip_missing_foreign_keys). +Table with a [sequence](create-sequence.html) | The sequence it uses; however, this dependency can be [removed during the restore](restore.html#skip_missing_sequences). +[Views](views.html) | The tables used in the view's `SELECT` statement. +[Interleaved tables](interleave-in-parent.html) | The parent table in the [interleaved hierarchy](interleave-in-parent.html#interleaved-hierarchy). + +### Users and privileges + +The `system.users` table stores your users and their passwords. To restore your users and privilege [grants](grant.html), do a full cluster backup and restore the cluster to a fresh cluster with no user data. You can also backup the `system.users` table, and then use [this procedure](backup-and-restore-advanced-options.html#restoring-users-from-system-users-backup). + +### Backup types + +CockroachDB offers two types of backups: [full](#full-backups) and [incremental](#incremental-backups). + +#### Full backups + +Full backups contain an unreplicated copy of your data and can always be used to restore your cluster. These files are roughly the size of your data and require greater resources to produce than incremental backups. You can take full backups as of a given timestamp and (optionally) include the available [revision history](backup-and-restore-advanced-options.html#backup-with-revision-history-and-point-in-time-restore). + +#### Incremental backups + +Incremental backups are smaller and faster to produce than full backups because they contain only the data that has changed since a base set of backups you specify (which must include one full backup, and can include many incremental backups). You can take incremental backups either as of a given timestamp or with full [revision history](backup-and-restore-advanced-options.html#backup-with-revision-history-and-point-in-time-restore). + +{{site.data.alerts.callout_danger}} +Incremental backups can only be created within the garbage collection period of the base backup's most recent timestamp. This is because incremental backups are created by finding which data has been created or modified since the most recent timestamp in the base backup––that timestamp data, though, is deleted by the garbage collection process. + +You can configure garbage collection periods using the `ttlseconds` [replication zone setting](configure-replication-zones.html). +{{site.data.alerts.end}} + +For an example of an incremental backup, see the [Create incremental backups](#create-incremental-backups) example below. + +## Performance + +The `BACKUP` process minimizes its impact to the cluster's performance by distributing work to all nodes. Each node backs up only a specific subset of the data it stores (those for which it serves writes; more details about this architectural concept forthcoming), with no two nodes backing up the same data. + +For best performance, we also recommend always starting backups with a specific [timestamp](timestamp.html) at least 10 seconds in the past. For example: + +~~~ sql +> BACKUP...AS OF SYSTEM TIME '-10s'; +~~~ + +This improves performance by decreasing the likelihood that the `BACKUP` will be [retried because it contends with other statements/transactions](transactions.html#transaction-retries). However, because `AS OF SYSTEM TIME` returns historical data, your reads might be stale. + +## Viewing and controlling backups jobs + +After CockroachDB successfully initiates a backup, it registers the backup as a job, and you can do the following: + + Action | SQL Statement +-----------------------+----------------- +View the backup status | [`SHOW JOBS`](show-jobs.html) +Pause the backup | [`PAUSE JOB`](pause-job.html) +Resume the backup | [`RESUME JOB`](resume-job.html) +Cancel the backup | [`CANCEL JOB`](cancel-job.html) + +You can also visit the [**Jobs** page](admin-ui-jobs-page.html) of the Admin UI to view job details. The `BACKUP` statement will return when the backup is finished or if it encounters an error. + +{{site.data.alerts.callout_info}} +The presence of a `BACKUP-CHECKPOINT` file in the backup destination usually means the backup is not complete. This file is created when a backup is initiated, and is replaced with a `BACKUP` file once the backup is finished. +{{site.data.alerts.end}} + +## Examples + +Per our guidance in the [Performance](#performance) section, we recommend starting backups from a time at least 10 seconds in the past using [`AS OF SYSTEM TIME`](as-of-system-time.html). Each example below follows this guidance. + +### Backup a cluster + + To backup a full cluster: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +'gs://acme-co-backup/test-cluster' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +### Backup a database + +To backup a single database: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP DATABASE bank \ +TO 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +To backup multiple databases: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP DATABASE bank, employees \ +TO 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +### Backup a table or view + +To backup a single table or view: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP bank.customers \ +TO 'gs://acme-co-backup/bank-customers-2017-03-27-weekly' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +To backup multiple tables: + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP bank.customers, bank.accounts \ +TO 'gs://acme-co-backup/database-bank-2017-03-27-weekly' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +### Create incremental backups + + If you backup to a destination already containing a full backup, an incremental backup will be produced in a subdirectory with a date-based name (e.g., `destination/day/time_1`, `destination/day/time_2`): + +{% include copy-clipboard.html %} +~~~ sql +> BACKUP TO \ +'gs://acme-co-backup/test-cluster' \ +AS OF SYSTEM TIME '-10s'; +~~~ + +{{site.data.alerts.callout_info}} +This incremental backup syntax does not work for backups using HTTP storage; you must [explicitly control where your incremental backups go](backup-and-restore-advanced-options.html#incremental-backups-with-explicitly-specified-destinations) by using the [`INCREMENTAL FROM` syntax](#synopsis). +{{site.data.alerts.end}} + +### Advanced examples + +{% include {{ page.version.version }}/backups/advanced-examples-list.md %} + +## See also + +- [Backup and Restore Data](backup-and-restore.html) +- [Back up and Restore Data - Advanced Options](backup-and-restore-advanced-options.html) +- [`SHOW BACKUP`](show-backup.html) +- [`RESTORE`](restore.html) +- [Configure Replication Zones](configure-replication-zones.html) + + + +[backup]: backup.html +[restore]: restore.html diff --git a/v20.2/begin-transaction.md b/v20.2/begin-transaction.md new file mode 100644 index 00000000000..e0f94bab96b --- /dev/null +++ b/v20.2/begin-transaction.md @@ -0,0 +1,170 @@ +--- +title: BEGIN +summary: Initiate a SQL transaction with the BEGIN statement in CockroachDB. +toc: true +--- + +The `BEGIN` [statement](sql-statements.html) initiates a [transaction](transactions.html), which either successfully executes all of the statements it contains or none at all. + +{{site.data.alerts.callout_danger}} +When using transactions, your application should include logic to [retry transactions](transactions.html#transaction-retries) that are aborted to break a dependency cycle between concurrent transactions. +{{site.data.alerts.end}} + + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/begin_transaction.html %} +
+ +## Required privileges + +No [privileges](authorization.html#assign-privileges) are required to initiate a transaction. However, privileges are required for each statement within a transaction. + +## Aliases + +In CockroachDB, the following are aliases for the `BEGIN` statement: + +- `BEGIN TRANSACTION` +- `START TRANSACTION` + +## Parameters + + Parameter | Description +-----------|------------- +`PRIORITY` | If you do not want the transaction to run with `NORMAL` priority, you can set it to `LOW` or `HIGH`.

Transactions with higher priority are less likely to need to be retried.

For more information, see [Transactions: Priorities](transactions.html#transaction-priorities).

**Default**: `NORMAL` +`READ` | Set the transaction access mode to `READ ONLY` or `READ WRITE`. The current transaction access mode is also exposed as the [session variable](show-vars.html) `transaction_read_only`.

**Default**: `READ WRITE` +`AS OF SYSTEM TIME` | Execute the transaction using the database contents "as of" a specified time in the past.

The `AS OF SYSTEM TIME` clause can be used only when the transaction is read-only. If the transaction contains any writes, or if the `READ WRITE` mode is specified, an error will be returned.

For more information, see [AS OF SYSTEM TIME](as-of-system-time.html).

+ + CockroachDB now only supports `SERIALIZABLE` isolation, so transactions can no longer be meaningfully set to any other `ISOLATION LEVEL`. In previous versions of CockroachDB, you could set transactions to `SNAPSHOT` isolation, but that feature has been removed. + +## Examples + +### Begin a transaction + +#### Use default settings + +Without modifying the `BEGIN` statement, the transaction uses `SERIALIZABLE` isolation and `NORMAL` priority. + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE products SET inventory = 0 WHERE sku = '8675309'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders (customer, sku, status) VALUES (1001, '8675309', 'new'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> RELEASE SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> COMMIT; +~~~ + +{{site.data.alerts.callout_danger}}This example assumes you're using client-side intervention to handle transaction retries.{{site.data.alerts.end}} + +#### Change priority + +You can set a transaction's priority to `LOW` or `HIGH`. + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN PRIORITY HIGH; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE products SET inventory = 0 WHERE sku = '8675309'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders (customer, sku, status) VALUES (1001, '8675309', 'new'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> RELEASE SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> COMMIT; +~~~ + +You can also set a transaction's priority with [`SET TRANSACTION`](set-transaction.html). + +{{site.data.alerts.callout_danger}} +This example assumes you're using [client-side intervention to handle transaction retries](transactions.html#client-side-intervention). +{{site.data.alerts.end}} + +### Use the `AS OF SYSTEM TIME` option + +You can execute the transaction using the database contents "as of" a specified time in the past. + +{% include {{ page.version.version }}/sql/begin-transaction-as-of-system-time-example.md %} + +{{site.data.alerts.callout_success}} +You can also use the [`SET TRANSACTION`](set-transaction.html#use-the-as-of-system-time-option) statement inside the transaction to achieve the same results. This syntax is easier to use from [drivers and ORMs](install-client-drivers.html). +{{site.data.alerts.end}} + +### Begin a transaction with automatic retries + +CockroachDB will [automatically retry](transactions.html#transaction-retries) all transactions that contain both `BEGIN` and `COMMIT` in the same batch. Batching is controlled by your driver or client's behavior, but means that CockroachDB receives all of the statements as a single unit, instead of a number of requests. + +From the perspective of CockroachDB, a transaction sent as a batch looks like this: + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; + +> DELETE FROM customers WHERE id = 1; + +> DELETE orders WHERE customer = 1; + +> COMMIT; +~~~ + +However, in your application's code, batched transactions are often just multiple statements sent at once. For example, in Go, this transaction would sent as a single batch (and automatically retried): + +~~~ go +db.Exec( + "BEGIN; + + DELETE FROM customers WHERE id = 1; + + DELETE orders WHERE customer = 1; + + COMMIT;" +) +~~~ + +Issuing statements this way signals to CockroachDB that you do not need to change any of the statement's values if the transaction doesn't immediately succeed, so it can continually retry the transaction until it's accepted. + +## See also + +- [Transactions](transactions.html) +- [`COMMIT`](commit-transaction.html) +- [`SAVEPOINT`](savepoint.html) +- [`RELEASE SAVEPOINT`](release-savepoint.html) +- [`ROLLBACK`](rollback-transaction.html) diff --git a/v20.2/bit.md b/v20.2/bit.md new file mode 100644 index 00000000000..3f391465ac4 --- /dev/null +++ b/v20.2/bit.md @@ -0,0 +1,126 @@ +--- +title: BIT +summary: The BIT and BIT VARYING data types stores bit arrays. +toc: true +--- + +The `BIT` and `VARBIT` [data types](data-types.html) stores bit arrays. +With `BIT`, the length is fixed; with `VARBIT`, the length can be variable. + +{% include {{page.version.version}}/sql/vectorized-support.md %} + +## Aliases + +The name `BIT VARYING` is an alias for `VARBIT`. + +## Syntax + +Bit array constants are expressed as literals. For example, `B'100101'` denotes an array of 6 bits. + +For more information about bit array constants, see the [constants documentation on bit array literals](sql-constants.html#bit-array-literals). + +For usage, see the [Example](#example) below. + +## Size + +The number of bits in a `BIT` value is determined as follows: + +| Type declaration | Logical size | +|------------------|-----------------------------------| +| BIT | 1 bit | +| BIT(N) | N bits | +| VARBIT | variable with no maximum | +| VARBIT(N) | variable with a maximum of N bits | + +The effective size of a `BIT` value is larger than its logical number +of bits by a bounded constant factor. Internally, CockroachDB stores +bit arrays in increments of 64 bits plus an extra integer value to +encode the length. + +The total size of a `BIT` value can be arbitrarily large, but it is +recommended to keep values under 1 MB to ensure performance. Above +that threshold, [write +amplification](https://en.wikipedia.org/wiki/Write_amplification) and +other considerations may cause significant performance degradation. + +## Example + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE b (x BIT, y BIT(3), z VARBIT, w VARBIT(3)); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM b; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden ++-------------+-----------+-------------+----------------+-----------------------+-----------+-----------+ + x | BIT | true | NULL | | {} | false + y | BIT(3) | true | NULL | | {} | false + z | VARBIT | true | NULL | | {} | false + w | VARBIT(3) | true | NULL | | {} | false + rowid | INT | false | unique_rowid() | | {primary} | true +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO b(x, y, z, w) VALUES (B'1', B'101', B'1', B'1'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM b; +~~~ + +~~~ + x | y | z | w ++---+-----+---+---+ + 1 | 101 | 1 | 1 +~~~ + +For type `BIT`, the value must match exactly the specified size: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO b(x) VALUES (B'101'); +~~~ + +~~~ +pq: bit string length 3 does not match type BIT +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO b(y) VALUES (B'10'); +~~~ + +~~~ +pq: bit string length 2 does not match type BIT(3) +~~~ + +For type `VARBIT`, the value must not be larger than the specified maximum size: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO b(w) VALUES (B'1010'); +~~~ + +~~~ +pq: bit string length 4 too large for type VARBIT(3) +~~~ + +## Supported casting and conversion + +`BIT` values can be [cast](data-types.html#data-type-conversions-and-casts) to any of the following data types: + +Type | Details +-----|--------- +`INT` | Converts the bit array to the corresponding numeric value, interpreting the bits as if the value was encoded using [two's complement](https://en.wikipedia.org/wiki/Two%27s_complement). If the bit array is larger than the integer type, excess bits on the left are ignored. For example, `B'1010'::INT` equals 10. +`STRING` | Prints out the binary digits as a string. This recovers the literal representation. For example, `B'1010'::INT` equals `'1010'`. + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/bool.md b/v20.2/bool.md new file mode 100644 index 00000000000..fbd5e72f4fa --- /dev/null +++ b/v20.2/bool.md @@ -0,0 +1,81 @@ +--- +title: BOOL +summary: The BOOL data type stores Boolean values of false or true. +toc: true +--- + +The `BOOL` [data type](data-types.html) stores a Boolean value of `false` or `true`. + + +## Aliases + +In CockroachDB, `BOOLEAN` is an alias for `BOOL`. + +## Syntax + +There are two predefined [named constants](sql-constants.html#named-constants) for `BOOL`: `TRUE` and `FALSE` (the names are case-insensitive). + +Alternately, a boolean value can be obtained by coercing a numeric value: zero is coerced to `FALSE`, and any non-zero value to `TRUE`. + +- `CAST(0 AS BOOL)` (false) +- `CAST(123 AS BOOL)` (true) + +## Size + +A `BOOL` value is 1 byte in width, but the total storage size is likely to be larger due to CockroachDB metadata. + +## Examples + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE bool (a INT PRIMARY KEY, b BOOL, c BOOLEAN); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM bool; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| a | INT | false | NULL | | {"primary"} | +| b | BOOL | true | NULL | | {} | +| c | BOOL | true | NULL | | {} | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO bool VALUES (12345, true, CAST(0 AS BOOL)); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM bool; +~~~ + +~~~ ++-------+------+-------+ +| a | b | c | ++-------+------+-------+ +| 12345 | true | false | ++-------+------+-------+ +~~~ + +## Supported casting and conversion + +`BOOL` values can be [cast](data-types.html#data-type-conversions-and-casts) to any of the following data types: + +Type | Details +-----|-------- +`INT` | Converts `true` to `1`, `false` to `0` +`DECIMAL` | Converts `true` to `1`, `false` to `0` +`FLOAT` | Converts `true` to `1`, `false` to `0` +`STRING` | –– + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/build-a-c++-app-with-cockroachdb.md b/v20.2/build-a-c++-app-with-cockroachdb.md new file mode 100644 index 00000000000..3c62814e96f --- /dev/null +++ b/v20.2/build-a-c++-app-with-cockroachdb.md @@ -0,0 +1,200 @@ +--- +title: Build a C++ App with CockroachDB and libpqxx +summary: Learn how to use CockroachDB from a simple C++ application with a low-level client driver. +toc: true +twitter: false +--- + +This tutorial shows you how build a simple C++ application with CockroachDB and the C++ libpqxx driver. + +We have tested the [C++ libpqxx driver](https://github.com/jtv/libpqxx) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the libpqxx driver + +Install the C++ libpqxx driver as described in the [official documentation](https://github.com/jtv/libpqxx). + +{{site.data.alerts.callout_info}} +If you are running macOS, you need to install version 4.0.1 or higher of the libpqxx driver. +{{site.data.alerts.end}} + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the C++ code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.cpp file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ cpp +{% include {{ page.version.version }}/app/basic-sample.cpp %} +~~~ + +To build the `basic-sample.cpp` source code to an executable file named `basic-sample`, run the following command from the directory that contains the code: + +{% include copy-clipboard.html %} +~~~ shell +$ g++ -std=c++11 basic-sample.cpp -lpq -lpqxx -o basic-sample +~~~ + +Then run the `basic-sample` file from that directory: + +{% include copy-clipboard.html %} +~~~ shell +$ ./basic-sample +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +Download the txn-sample.cpp file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ cpp +{% include {{ page.version.version }}/app/txn-sample.cpp %} +~~~ + +To build the `txn-sample.cpp` source code to an executable file named `txn-sample`, run the following command from the directory that contains the code: + +{% include copy-clipboard.html %} +~~~ shell +$ g++ -std=c++11 txn-sample.cpp -lpq -lpqxx -o txn-sample +~~~ + +Then run the `txn-sample` file from that directory: + +{% include copy-clipboard.html %} +~~~ shell +$ ./txn-sample +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ +id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the C++ code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.cpp file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ cpp +{% include {{ page.version.version }}/app/insecure/basic-sample.cpp %} +~~~ + +To build the `basic-sample.cpp` source code to an executable file named `basic-sample`, run the following command from the directory that contains the code: + +{% include copy-clipboard.html %} +~~~ shell +$ g++ -std=c++11 basic-sample.cpp -lpq -lpqxx -o basic-sample +~~~ + +Then run the `basic-sample` file from that directory: + +{% include copy-clipboard.html %} +~~~ shell +$ ./basic-sample +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +Download the txn-sample.cpp file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ cpp +{% include {{ page.version.version }}/app/insecure/txn-sample.cpp %} +~~~ + +To build the `txn-sample.cpp` source code to an executable file named `txn-sample`, run the following command from the directory that contains the code: + +{% include copy-clipboard.html %} +~~~ shell +$ g++ -std=c++11 txn-sample.cpp -lpq -lpqxx -o txn-sample +~~~ + +Then run the `txn-sample` file from that directory: + +{% include copy-clipboard.html %} +~~~ shell +$ ./txn-sample +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ +id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [C++ libpqxx driver](https://github.com/jtv/libpqxx). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-clojure-app-with-cockroachdb.md b/v20.2/build-a-clojure-app-with-cockroachdb.md new file mode 100644 index 00000000000..5b8a9347e4c --- /dev/null +++ b/v20.2/build-a-clojure-app-with-cockroachdb.md @@ -0,0 +1,222 @@ +--- +title: Build a Clojure App with CockroachDB and java.jdbc +summary: Learn how to use CockroachDB from a simple Clojure application with a low-level client driver. +toc: true +twitter: false +--- + +This tutorial shows you how build a simple Clojure application with CockroachDB using [leiningen](https://leiningen.org/) and the Closure java.jdbc driver. + +We have tested the [Clojure java.jdbc driver](http://clojure-doc.org/articles/ecosystem/java_jdbc/home.html) in conjunction with the [PostgreSQL JDBC driver](https://jdbc.postgresql.org/) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install `leiningen` + +Install the Clojure `lein` utility as described in its [official documentation](https://leiningen.org/). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +New in v19.1: Pass the [`--also-generate-pkcs8-key` flag](cockroach-cert.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key --also-generate-pkcs8-key +~~~ + +## Step 4. Create a table in the new database + +As the `maxroach` user, use the [built-in SQL client](cockroach-sql.html) to create an `accounts` table in the new database. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql \ +--certs-dir=certs \ +--database=bank \ +--user=maxroach \ +-e 'CREATE TABLE accounts (id INT PRIMARY KEY, balance INT)' +~~~ + +## Step 5. Run the Clojure code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Create a basic Clojure/JDBC project + +1. Create a new directory `myapp`. +2. Create a file `myapp/project.clj` and populate it with the following code, or download it directly. + + {% include copy-clipboard.html %} + ~~~ clojure + {% include {{ page.version.version }}/app/project.clj %} + ~~~ + +3. Create a file `myapp/src/test/util.clj` and populate it with the code from this file. Be sure to place the file in the subdirectory `src/test` in your project. + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Create a file `myapp/src/test/test.clj` and copy the code below to it, or download it directly. Be sure to rename this file to `test.clj` in the subdirectory `src/test` in your project. + +{% include copy-clipboard.html %} +~~~ clojure +{% include {{ page.version.version }}/app/basic-sample.clj %} +~~~ + +Run with: + +{% include copy-clipboard.html %} +~~~ shell +$ lein run +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Copy the code below to `myapp/src/test/test.clj` or +download it directly. Again, preserve the file name `test.clj`. + +{{site.data.alerts.callout_info}} +CockroachDB may require the +[client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ clojure +{% include {{ page.version.version }}/app/txn-sample.clj %} +~~~ + +Run with: + +{% include copy-clipboard.html %} +~~~ shell +$ lein run +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ +id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Create a table in the new database + +As the `maxroach` user, use the [built-in SQL client](cockroach-sql.html) to create an `accounts` table in the new database. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--database=bank \ +--user=maxroach \ +-e 'CREATE TABLE accounts (id INT PRIMARY KEY, balance INT)' +~~~ + +## Step 4. Run the Clojure code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Create a basic Clojure/JDBC project + +1. Create a new directory `myapp`. +2. Create a file `myapp/project.clj` and populate it with the following code, or download it directly. + + {% include copy-clipboard.html %} + ~~~ clojure + {% include {{ page.version.version }}/app/project.clj %} + ~~~ + +3. Create a file `myapp/src/test/util.clj` and populate it with the code from this file. Be sure to place the file in the subdirectory `src/test` in your project. + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Create a file `myapp/src/test/test.clj` and copy the code below to it, or download it directly. Be sure to rename this file to `test.clj` in the subdirectory `src/test` in your project. + +{% include copy-clipboard.html %} +~~~ clojure +{% include {{ page.version.version }}/app/insecure/basic-sample.clj %} +~~~ + +Run with: + +{% include copy-clipboard.html %} +~~~ shell +$ lein run +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Copy the code below to `myapp/src/test/test.clj` or +download it directly. Again, preserve the file name `test.clj`. + +{{site.data.alerts.callout_info}} +CockroachDB may require the +[client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ clojure +{% include {{ page.version.version }}/app/insecure/txn-sample.clj %} +~~~ + +Run with: + +{% include copy-clipboard.html %} +~~~ shell +$ lein run +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ +id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Clojure java.jdbc driver](http://clojure-doc.org/articles/ecosystem/java_jdbc/home.html). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-csharp-app-with-cockroachdb.md b/v20.2/build-a-csharp-app-with-cockroachdb.md new file mode 100644 index 00000000000..604516658c5 --- /dev/null +++ b/v20.2/build-a-csharp-app-with-cockroachdb.md @@ -0,0 +1,235 @@ +--- +title: Build a C# App with CockroachDB and the .NET Npgsql Driver +summary: Learn how to use CockroachDB from a simple C# (.NET) application with a low-level client driver. +toc: true +twitter: true +--- + +This tutorial shows you how build a simple C# application with CockroachDB and the .NET Npgsql driver. + +We have tested the [.NET Npgsql driver](http://www.npgsql.org/) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Create a .NET project + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet new console -o cockroachdb-test-app +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cd cockroachdb-test-app +~~~ + +The `dotnet` command creates a new app of type `console`. The `-o` parameter creates a directory named `cockroachdb-test-app` where your app will be stored and populates it with the required files. The `cd cockroachdb-test-app` command puts you into the newly created app directory. + +## Step 2. Install the Npgsql driver + +Install the latest version of the [Npgsql driver](https://www.nuget.org/packages/Npgsql/) into the .NET project using the built-in nuget package manager: + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet add package Npgsql +~~~ + +
+ +## Step 3. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 4. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 5. Convert the key file for use by C# programs + +The private key generated for user `maxroach` by CockroachDB is [PEM encoded](https://tools.ietf.org/html/rfc1421). To read the key in a C# application, you will need to convert it into PKCS#12 format. + +To convert the key to PKCS#12 format, run the following OpenSSL command on the `maxroach` user's key file in the directory where you stored your certificates: + +{% include copy-clipboard.html %} +~~~ shell +$ openssl pkcs12 -inkey client.maxroach.key -password pass: -in client.maxroach.crt -export -out client.maxroach.pfx +~~~ + +As of December 2018, you need to provide a password for this to work on macOS. See . + +## Step 6. Run the C# code + +Now that you have created a database and set up encryption keys, in this section you will: + +- [Create a table and insert some rows](#basic-example) +- [Execute a batch of statements as a transaction](#transaction-example-with-retry-logic) + +### Basic example + +Replace the contents of `cockroachdb-test-app/Program.cs` with the following code: + +{% include copy-clipboard.html %} +~~~ csharp +{% include {{ page.version.version }}/app/basic-sample.cs %} +~~~ + +Then, run the code to connect as the `maxroach` user. This time, execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted: + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet run +~~~ + +The output should be: + +~~~ +Initial balances: + account 1: 1000 + account 2: 250 +~~~ + +### Transaction example (with retry logic) + +Open `cockroachdb-test-app/Program.cs` again and replace the contents with the code shown below. + +{% include {{page.version.version}}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ csharp +{% include {{ page.version.version }}/app/txn-sample.cs %} +~~~ + +Then, run the code to connect as the `maxroach` user. This time, execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted: + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet run +~~~ + +The output should be: + +~~~ +Initial balances: + account 1: 1000 + account 2: 250 +Final balances: + account 1: 900 + account 2: 350 +~~~ + +However, if you want to verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank -e 'SELECT id, balance FROM accounts' +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + +
+ +
+ +## Step 3. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 4. Run the C# code + +Now that you have created a database and set up encryption keys, in this section you will: + +- [Create a table and insert some rows](#basic2) +- [Execute a batch of statements as a transaction](#transaction2) + + + +### Basic example + +Replace the contents of `cockroachdb-test-app/Program.cs` with the following code: + +{% include copy-clipboard.html %} +~~~ csharp +{% include {{ page.version.version }}/app/insecure/basic-sample.cs %} +~~~ + +Then, run the code to connect as the `maxroach` user and execute some basic SQL statements: creating a table, inserting rows, and reading and printing the rows: + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet run +~~~ + +The output should be: + +~~~ +Initial balances: + account 1: 1000 + account 2: 250 +~~~ + + + +### Transaction example (with retry logic) + +Open `cockroachdb-test-app/Program.cs` again and replace the contents with the code shown below. + +{% include {{page.version.version}}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ csharp +{% include {{ page.version.version }}/app/insecure/txn-sample.cs %} +~~~ + +Then, run the code to connect as the `maxroach` user. This time, execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted: + +{% include copy-clipboard.html %} +~~~ shell +$ dotnet run +~~~ + +The output should be: + +~~~ +Initial balances: + account 1: 1000 + account 2: 250 +Final balances: + account 1: 900 + account 2: 350 +~~~ + +However, if you want to verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank -e 'SELECT id, balance FROM accounts' +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [.NET Npgsql driver](http://www.npgsql.org/). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-go-app-with-cockroachdb-gorm.md b/v20.2/build-a-go-app-with-cockroachdb-gorm.md new file mode 100644 index 00000000000..95047fd5c85 --- /dev/null +++ b/v20.2/build-a-go-app-with-cockroachdb-gorm.md @@ -0,0 +1,148 @@ +--- +title: Build a Go App with CockroachDB and GORM +summary: Learn how to use CockroachDB from a simple Go application with the GORM ORM. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Go application with CockroachDB and the GORM ORM. + +We have tested the [GORM ORM](http://gorm.io) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +{{site.data.alerts.callout_success}} +For another use of GORM with CockroachDB, see our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. +{{site.data.alerts.end}} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the GORM ORM + +To install [GORM](http://gorm.io), run the following commands: + +{% include copy-clipboard.html %} +~~~ shell +$ go get -u github.com/lib/pq # dependency +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ go get -u github.com/jinzhu/gorm +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Go code + +The following code uses the [GORM](http://gorm.io) ORM to map Go-specific objects to SQL operations. Specifically: + +- `db.AutoMigrate(&Account{})` creates an `accounts` table based on the Account model. +- `db.Create(&Account{})` inserts rows into the table. +- `db.Find(&accounts)` selects from the table so that balances can be printed. +- The funds transfer occurs in `transferFunds()`. To ensure that we [handle retry errors](transactions.html#client-side-intervention), we write an application-level retry loop that, in case of error, sleeps before trying the funds transfer again. If it encounters another error, it sleeps again for a longer interval, implementing [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff). + +Copy the code or +download it directly. + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-go-gorm/` +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/gorm-sample.go %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run gorm-sample.go +~~~ + +The output should show the account balances before and after the funds transfer: + +~~~ shell +Balance at '2019-08-06 13:37:19.311423 -0400 EDT m=+0.034072606': +1 1000 +2 250 +Balance at '2019-08-06 13:37:19.325654 -0400 EDT m=+0.048303286': +1 900 +2 350 +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Go code + +The following code uses the [GORM](http://gorm.io) ORM to map Go-specific objects to SQL operations. Specifically: + +- `db.AutoMigrate(&Account{})` creates an `accounts` table based on the Account model. +- `db.Create(&Account{})` inserts rows into the table. +- `db.Find(&accounts)` selects from the table so that balances can be printed. +- The funds transfer occurs in `transferFunds()`. To ensure that we [handle retry errors](transactions.html#client-side-intervention), we write an application-level retry loop that, in case of error, sleeps before trying the funds transfer again. If it encounters another error, it sleeps again for a longer interval, implementing [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff). + +To get the code below, clone the `hello-world-go-gorm` repo to your machine: + +{% include copy-clipboard.html %} +~~~ shell +git clone https://github.com/cockroachlabs/hello-world-go-gorm/ +~~~ + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/insecure/gorm-sample.go %} +~~~ + +Change to the directory where you cloned the repo and run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run main.go +~~~ + +The output should show the account balances before and after the funds transfer: + +~~~ shell +Balance at '2019-07-15 13:34:22.536363 -0400 EDT m=+0.019918599': +1 1000 +2 250 +Balance at '2019-07-15 13:34:22.540037 -0400 EDT m=+0.023592845': +1 900 +2 350 +~~~ + +
+ +## What's next? + +Read more about using the [GORM ORM](http://gorm.io), or check out a more realistic implementation of GORM with CockroachDB in our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-go-app-with-cockroachdb.md b/v20.2/build-a-go-app-with-cockroachdb.md new file mode 100644 index 00000000000..0669616836a --- /dev/null +++ b/v20.2/build-a-go-app-with-cockroachdb.md @@ -0,0 +1,239 @@ +--- +title: Build a Go App with CockroachDB the Go pq Driver +summary: Learn how to use CockroachDB from a simple Go application with the Go pq driver. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Go application with CockroachDB and the Go pq driver. + +We have tested the [Go pq driver](https://godoc.org/github.com/lib/pq) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the Go pq driver + +To install the [Go pq driver](https://godoc.org/github.com/lib/pq), run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ go get -u github.com/lib/pq +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Go code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-go-pq/` +{{site.data.alerts.end}} + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.go file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/basic-sample.go %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run basic-sample.go +~~~ + +The output should be: + +~~~ +Initial balances: +1 1000 +2 250 +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time will execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.go file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/txn-sample.go %} +~~~ + +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. For Go, the CockroachDB retry function is in the `crdb` package of the CockroachDB Go client. To install Clone the library into your `$GOPATH` as follows: + +{% include copy-clipboard.html %} +~~~ shell +$ mkdir -p $GOPATH/src/github.com/cockroachdb +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cd $GOPATH/src/github.com/cockroachdb +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ git clone git@github.com:cockroachdb/cockroach-go.git +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run txn-sample.go +~~~ + +The output should be: + +~~~ shell +Success +~~~ + +To verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Go code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-go-pq/` +{{site.data.alerts.end}} + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.go file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/insecure/basic-sample.go %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run basic-sample.go +~~~ + +The output should be: + +~~~ +Initial balances: +1 1000 +2 250 +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time will execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.go file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/app/insecure/txn-sample.go %} +~~~ + +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. For Go, the CockroachDB retry function is in the `crdb` package of the CockroachDB Go client. + +To install the [CockroachDB Go client](https://github.com/cockroachdb/cockroach-go), run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ go get -d github.com/cockroachdb/cockroach-go +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ go run txn-sample.go +~~~ + +The output should be: + +~~~ shell +Success +~~~ + +To verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Go pq driver](https://godoc.org/github.com/lib/pq). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-java-app-with-cockroachdb-hibernate.md b/v20.2/build-a-java-app-with-cockroachdb-hibernate.md new file mode 100644 index 00000000000..8f0ea876e0a --- /dev/null +++ b/v20.2/build-a-java-app-with-cockroachdb-hibernate.md @@ -0,0 +1,316 @@ +--- +title: Build a Java App with CockroachDB and Hibernate +summary: Learn how to use CockroachDB from a simple Java application with the Hibernate ORM. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Java application with CockroachDB and the Hibernate ORM. + +{{site.data.alerts.callout_success}} +For another use of Hibernate with CockroachDB, see our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. +{{site.data.alerts.end}} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +{{site.data.alerts.callout_danger}} +The examples on this page assume you are using a Java version <= 9. They do not work with Java 10. +{{site.data.alerts.end}} + +## Step 1. Install the Gradle build tool + +This tutorial uses the [Gradle build tool](https://gradle.org/) to get all dependencies for your application, including Hibernate. + +To install Gradle on Mac, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ brew install gradle +~~~ + +To install Gradle on a Debian-based Linux distribution like Ubuntu: + +{% include copy-clipboard.html %} +~~~ shell +$ apt-get install gradle +~~~ + +To install Gradle on a Red Hat-based Linux distribution like Fedora: + +{% include copy-clipboard.html %} +~~~ shell +$ dnf install gradle +~~~ + +For other ways to install Gradle, see [its official documentation](https://gradle.org/install). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +You can pass the [`--also-generate-pkcs8-key` flag](cockroach-cert.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key --also-generate-pkcs8-key +~~~ + +## Step 4. Run the Java code + +The code below uses Hibernate to map Java methods to SQL operations. It perform the following steps which roughly correspond to method calls in the `Sample` class. + +1. Create an `accounts` table in the `bank` database as specified by the Hibernate `Account` class. +2. Inserts rows into the table using `session.save(new Account(int id, int balance))` (see `Sample.addAccounts()`). +3. Transfer money from one account to another, printing out account balances before and after the transfer (see `transferFunds(long fromId, long toId, long amount)`). +4. Print out account balances before and after the transfer (see `Sample.getAccountBalance(long id)`). + +In addition, the code shows a pattern for automatically handling [transaction retries](transactions.html#client-side-intervention-example) by wrapping transactions in a higher-order function `Sample.runTransaction()`. It also includes a method for testing the retry handling logic (`Sample.forceRetryLogic()`), which will be run if you set the `FORCE_RETRY` variable to `true`. + +It does all of the above using the practices we recommend for using Hibernate (and the underlying JDBC connection) with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. + +To run it: + +1. Download and extract [hibernate-basic-sample.tgz](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/hibernate-basic-sample/hibernate-basic-sample.tgz). The settings in [`hibernate.cfg.xml`](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/hibernate-basic-sample/hibernate.cfg.xml) specify how to connect to the database. +2. Compile and run the code using [`build.gradle`](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/hibernate-basic-sample/build.gradle), which will also download the dependencies. + + {% include copy-clipboard.html %} + ~~~ shell + $ gradle run + ~~~ + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-java-hibernate/` +{{site.data.alerts.end}} + +The contents of [`Sample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{page.version.version}}/app/hibernate-basic-sample/Sample.java): + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/hibernate-basic-sample/Sample.java %} +~~~ + +Toward the end of the output, you should see: + +~~~ +APP: BEGIN; +APP: addAccounts() --> 1 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(1) --> 1000 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 250 +APP: COMMIT; +APP: getAccountBalance(1) --> 1000 +APP: getAccountBalance(2) --> 250 +APP: BEGIN; +APP: transferFunds(1, 2, 100) --> 100 +APP: COMMIT; +APP: transferFunds(1, 2, 100) --> 100 +APP: BEGIN; +APP: getAccountBalance(1) --> 900 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 350 +APP: COMMIT; +APP: getAccountBalance(1) --> 900 +APP: getAccountBalance(2) --> 350 +~~~ + +To verify that the account balances were updated successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +SELECT id, balance FROM accounts; +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 + 3 | 314159 +(3 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Java code + +The code below uses Hibernate to map Java methods to SQL operations. It perform the following steps which roughly correspond to method calls in the `Sample` class. + +1. Create an `accounts` table in the `bank` database as specified by the Hibernate `Account` class. +2. Inserts rows into the table using `session.save(new Account(int id, int balance))` (see `Sample.addAccounts()`). +3. Transfer money from one account to another, printing out account balances before and after the transfer (see `transferFunds(long fromId, long toId, long amount)`). +4. Print out account balances before and after the transfer (see `Sample.getAccountBalance(long id)`). + +In addition, the code shows a pattern for automatically handling [transaction retries](transactions.html#client-side-intervention-example) by wrapping transactions in a higher-order function `Sample.runTransaction()`. It also includes a method for testing the retry handling logic (`Sample.forceRetryLogic()`), which will be run if you set the `FORCE_RETRY` variable to `true`. + +It does all of the above using the practices we recommend for using Hibernate (and the underlying JDBC connection) with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. + +To run it: + +1. Clone the `hello-world-java-hibernate` repo to your machine: + + {% include copy-clipboard.html %} + ~~~ shell + git clone https://github.com/cockroachlabs/hello-world-java-hibernate/ + ~~~ + +2. Compile and run the code using [`build.gradle`](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/insecure/hibernate-basic-sample/build.gradle), which will also download the dependencies. + + {% include copy-clipboard.html %} + ~~~ shell + $ gradle run + ~~~ + +The contents of [`Sample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{page.version.version}}/app/insecure/hibernate-basic-sample/Sample.java): + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/insecure/hibernate-basic-sample/Sample.java %} +~~~ + +Toward the end of the output, you should see: + +~~~ +APP: BEGIN; +APP: addAccounts() --> 1 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(1) --> 1000 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 250 +APP: COMMIT; +APP: getAccountBalance(1) --> 1000 +APP: getAccountBalance(2) --> 250 +APP: BEGIN; +APP: transferFunds(1, 2, 100) --> 100 +APP: COMMIT; +APP: transferFunds(1, 2, 100) --> 100 +APP: BEGIN; +APP: getAccountBalance(1) --> 900 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 350 +APP: COMMIT; +APP: getAccountBalance(1) --> 900 +APP: getAccountBalance(2) --> 350 +~~~ + +To verify that the account balances were updated successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +SELECT id, balance FROM accounts; +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 + 3 | 314159 +(3 rows) +~~~ + +
+ +## Recommended Practices + +### Use `IMPORT` to read in large data sets + +If you are trying to get a large data set into CockroachDB all at once (a bulk import), avoid writing client-side code altogether and use the [`IMPORT`](import.html) statement instead. It is much faster and more efficient than making a series of [`INSERT`s](insert.html) and [`UPDATE`s](update.html). It bypasses the [SQL layer](architecture/sql-layer.html) altogether and writes directly to the [storage layer](architecture/storage-layer.html) of the database. + +For more information about importing data from Postgres, see [Migrate from Postgres](migrate-from-postgres.html). + +For more information about importing data from MySQL, see [Migrate from MySQL](migrate-from-mysql.html). + +### Use `rewriteBatchedInserts` for increased speed + +We strongly recommend setting `rewriteBatchedInserts=true`; we have seen 2-3x performance improvements with it enabled. From [the JDBC connection parameters documentation](https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters): + +> This will change batch inserts from `insert into foo (col1, col2, col3) values (1,2,3)` into `insert into foo (col1, col2, col3) values (1,2,3), (4,5,6)` this provides 2-3x performance improvement + +### Use a batch size of 128 + +PGJDBC's batching support only works with [powers of two](https://github.com/pgjdbc/pgjdbc/blob/7b52b0c9e5b9aa9a9c655bb68f23bf4ec57fd51c/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java#L1597), and will split batches of other sizes up into multiple sub-batches. This means that a batch of size 128 can be 6x faster than a batch of size 250. + +The code snippet below shows a pattern for using a batch size of 128, and is taken from the longer example above (specifically, the `BasicExampleDAO.bulkInsertRandomAccountData()` method). + +Specifically, it does the following: + +1. Turn off auto-commit so you can manage the transaction lifecycle and thus the size of the batch inserts. +2. Given an overall update size of 500 rows (for example), split it into batches of size 128 and execute each batch in turn. +3. Finally, commit the batches of statements you've just executed. + +~~~ java +int BATCH_SIZE = 128; +connection.setAutoCommit(false); + +try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); // Verifying 128 rows in the batch + } + connection.commit(); +} +~~~ + +### Retrieve large data sets in chunks using cursors + +CockroachDB now supports the Postgres wire-protocol cursors for implicit transactions and explicit transactions executed to completion. This means the [PGJDBC driver](https://jdbc.postgresql.org) can use this protocol to stream queries with large result sets. This is much faster than [paginating through results in SQL using `LIMIT .. OFFSET`](selection-queries.html#paginate-through-limited-results). + +For instructions showing how to use cursors in your Java code, see [Getting results based on a cursor](https://jdbc.postgresql.org/documentation/head/query.html#query-with-cursor) from the PGJDBC documentation. + +Note that interleaved execution (partial execution of multiple statements within the same connection and transaction) is not supported when [`Statement.setFetchSize()`](https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#setFetchSize-int-) is used. + +## What's next? + +Read more about using the [Hibernate ORM](http://hibernate.org/orm/), or check out a more realistic implementation of Hibernate with CockroachDB in our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-java-app-with-cockroachdb-jooq.md b/v20.2/build-a-java-app-with-cockroachdb-jooq.md new file mode 100644 index 00000000000..9bcfc327e23 --- /dev/null +++ b/v20.2/build-a-java-app-with-cockroachdb-jooq.md @@ -0,0 +1,277 @@ +--- +title: Build a Java App with CockroachDB and jOOQ +summary: Learn how to use CockroachDB from a simple Java application with jOOQ. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Java application with CockroachDB and [jOOQ](https://www.jooq.org/). + +CockroachDB is supported in jOOQ [Professional and Enterprise editions](https://www.jooq.org/download/#databases). + +{{site.data.alerts.callout_success}} +For another use of jOOQ with CockroachDB, see our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. +{{site.data.alerts.end}} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install Maven + +This tutorial uses the [Maven build tool](https://gradle.org/) to manage application dependencies. + +To install Maven on Mac, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ brew install maven +~~~ + +To install Maven on a Debian-based Linux distribution like Ubuntu: + +{% include copy-clipboard.html %} +~~~ shell +$ apt-get install maven +~~~ + +For other ways to install Maven, see [its official documentation](https://maven.apache.org/install.html). + +## Step 2. Install jOOQ + +Download the free trial of jOOQ Professional or Enterprise edition from [jOOQ's website](https://www.jooq.org/download), and unzip the file. + +{{site.data.alerts.callout_info}} +If you have Java 6, Java 8, or Java 11, go to [Download your jOOQ version](https://www.jooq.org/download/versions#trial), and download the free trial of jOOQ for your version of Java. +{{site.data.alerts.end}} + +To install jOOQ to your machine's local Maven repository, run the `maven-install.sh` script included in the jOOQ install folder: + +{% include copy-clipboard.html %} +~~~ shell +$ chmod +x maven-install.sh +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ ./maven-install.sh +~~~ + +
+ +## Step 3. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 4. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +The [`--also-generate-pkcs8-key` flag](cockroach-cert.html#flag-pkcs8) generates a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key --also-generate-pkcs8-key +~~~ + +## Step 5. Run the Java code + +The code below uses jOOQ to map Java methods to SQL operations. It performs the following steps, some of which correspond to method calls of the `Sample` class. + +1. Inputs the `db.sql` file to the database. `db.sql` includes SQL statements that create an `accounts` table in the `bank` database. +2. Inserts rows into the `accounts` table using `session.save(new Account(int id, int balance))` (see `Sample.addAccounts()`). +3. Transfers money from one account to another, printing out account balances before and after the transfer (see `transferFunds(long fromId, long toId, long amount)`). +4. Prints out account balances before and after the transfer (see `Sample.getAccountBalance(long id)`). + +In addition, the code shows a pattern for automatically handling [transaction retries](transactions.html#client-side-intervention-example) by wrapping transactions in a higher-order function `Sample.runTransaction()`. It also includes a method for testing the retry handling logic (`Sample.forceRetryLogic()`), which will be run if you set the `FORCE_RETRY` variable to `true`. + +To run it: + +1. Download and unzip [jooq-basic-sample.zip](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/jooq-basic-sample/jooq-basic-sample.zip). +2. Open `jooq-basic-sample/src/main/java/com/cockroachlabs/Sample.java`, and edit the connection string passed to `DriverManager.getConnection()` in the `Sample` class's `main()` method so that the certificate paths are fully and correctly specified. +3. Compile and run the code using Maven: + + {% include copy-clipboard.html %} + ~~~ shell + $ cd jooq-basic-sample + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mvn compile + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mvn exec:java -Dexec.mainClass=com.cockroachlabs.Sample + ~~~ + +Here are the contents of [`Sample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{page.version.version}}/app/jooq-basic-sample/Sample.java), the Java file containing the main `Sample` class: + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/jooq-basic-sample/Sample.java %} +~~~ + +Toward the end of the output, you should see: + +~~~ +APP: BEGIN; +APP: addAccounts() --> 1 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(1) --> 1000 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 250 +APP: COMMIT; +APP: getAccountBalance(1) --> 1000 +APP: getAccountBalance(2) --> 250 +APP: BEGIN; +APP: transferFunds(1, 2, 100) --> 100 +APP: COMMIT; +APP: transferFunds(1, 2, 100) --> 100 +APP: BEGIN; +APP: getAccountBalance(1) --> 900 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 350 +APP: COMMIT; +APP: getAccountBalance(1) --> 900 +APP: getAccountBalance(2) --> 350 +~~~ + +To verify that the account balances were updated successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +SELECT id, balance FROM accounts; +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 + 3 | 314159 +(3 rows) +~~~ + +
+ +
+ +## Step 3. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 4. Run the Java code + +The code below uses jOOQ to map Java methods to SQL operations. It performs the following steps, some of which correspond to method calls of the `Sample` class. + +1. Inputs the `db.sql` file to the database. `db.sql` includes SQL statements that create an `accounts` table in the `bank` database. +2. Inserts rows into the `accounts` table using `session.save(new Account(int id, int balance))` (see `Sample.addAccounts()`). +3. Transfers money from one account to another, printing out account balances before and after the transfer (see `transferFunds(long fromId, long toId, long amount)`). +4. Prints out account balances before and after the transfer (see `Sample.getAccountBalance(long id)`). + +In addition, the code shows a pattern for automatically handling [transaction retries](transactions.html#client-side-intervention-example) by wrapping transactions in a higher-order function `Sample.runTransaction()`. It also includes a method for testing the retry handling logic (`Sample.forceRetryLogic()`), which will be run if you set the `FORCE_RETRY` variable to `true`. + +To run it: + +1. Download and unzip [jooq-basic-sample.zip](https://github.com/cockroachdb/docs/raw/master/_includes/{{ page.version.version }}/app/insecure/jooq-basic-sample/jooq-basic-sample.zip). +2. Compile and run the code using Maven: + + {% include copy-clipboard.html %} + ~~~ shell + $ cd jooq-basic-sample + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mvn compile + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mvn exec:java -Dexec.mainClass=com.cockroachlabs.Sample + ~~~ + +Here are the contents of [`Sample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{page.version.version}}/app/insecure/jooq-basic-sample/Sample.java), the Java file containing the main `Sample` class: + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/insecure/jooq-basic-sample/Sample.java %} +~~~ + +Toward the end of the output, you should see: + +~~~ +APP: BEGIN; +APP: addAccounts() --> 1 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(1) --> 1000 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 250 +APP: COMMIT; +APP: getAccountBalance(1) --> 1000 +APP: getAccountBalance(2) --> 250 +APP: BEGIN; +APP: transferFunds(1, 2, 100) --> 100 +APP: COMMIT; +APP: transferFunds(1, 2, 100) --> 100 +APP: BEGIN; +APP: getAccountBalance(1) --> 900 +APP: COMMIT; +APP: BEGIN; +APP: getAccountBalance(2) --> 350 +APP: COMMIT; +APP: getAccountBalance(1) --> 900 +APP: getAccountBalance(2) --> 350 +~~~ + +To verify that the account balances were updated successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +SELECT id, balance FROM accounts; +~~~ + +~~~ + id | balance ++----+---------+ + 1 | 900 + 2 | 350 + 3 | 314159 +(3 rows) +~~~ + +
+ + +## What's next? + +Read more about using [jOOQ](https://www.jooq.org/), or check out a more realistic implementation of jOOQ with CockroachDB in our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-java-app-with-cockroachdb.md b/v20.2/build-a-java-app-with-cockroachdb.md new file mode 100644 index 00000000000..15eb9f7b4a4 --- /dev/null +++ b/v20.2/build-a-java-app-with-cockroachdb.md @@ -0,0 +1,283 @@ +--- +title: Build a Java App with CockroachDB and JDBC +summary: Learn how to use CockroachDB from a simple Java application with the JDBC driver. +toc: true +twitter: false +--- + + + +This tutorial shows you how to build a simple Java application with CockroachDB and the Java JDBC driver. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +{{site.data.alerts.callout_danger}} +The examples on this page assume you are using a Java version <= 9. They do not work with Java 10. +{{site.data.alerts.end}} + +## Step 1. Install the Java JDBC driver + +Download and set up the Java JDBC driver as described in the [official documentation](https://jdbc.postgresql.org/documentation/head/setup.html). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +New in v19.1: You can pass the [`--also-generate-pkcs8-key` flag](cockroach-cert.html#flag-pkcs8) to generate a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format in Java. In this case, the generated PKCS8 key will be named `client.maxroach.key.pk8`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key --also-generate-pkcs8-key +~~~ + +## Step 4. Run the Java code + +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: + +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). + +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. + +| Step | Method | +|------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------| +| 1. Create an `accounts` table in the `bank` database | `BasicExampleDAO.createAccounts()` | +| 2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend | `BasicExampleDAO.updateAccounts(Map balance)` | +| 3. Transfer money from one account to another, printing out account balances before and after the transfer | `BasicExampleDAO.transferFunds(int from, int to, int amount)` | +| 4. Insert random account data using JDBC's bulk insertion support | `BasicExampleDAO.bulkInsertRandomAccountData()` | +| 5. Print out some account data | `BasicExampleDAO.readAccounts(int limit)` | +| 6. Drop the `accounts` table and perform any other necessary cleanup | `BasicExampleDAO.tearDown()` (This cleanup step means you can run this program more than once.) | + +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. + +To run it: + +1. Download [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java), or create the file yourself and copy the code below. +2. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): + + {% include copy-clipboard.html %} + ~~~ shell + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ java -classpath .:/path/to/postgresql.jar BasicExample + ~~~ + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-java-jdbc/` +{{site.data.alerts.end}} + +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/BasicExample.java): + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/BasicExample.java %} +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Java code + +The code below uses JDBC and the [Data Access Object (DAO)](https://en.wikipedia.org/wiki/Data_access_object) pattern to map Java methods to SQL operations. It consists of two classes: + +1. `BasicExample`, which is where the application logic lives. +2. `BasicExampleDAO`, which is used by the application to access the data store (in this case CockroachDB). This class has logic to handle [transaction retries](transactions.html#transaction-retries) (see the `BasicExampleDAO.runSQL()` method). + +It performs the following steps which roughly correspond to method calls in the `BasicExample` class. + +1. Create an `accounts` table in the `bank` database (`BasicExampleDAO.createAccounts()`). +2. Insert account data using a `Map` that corresponds to the input to `INSERT` on the backend (`BasicExampleDAO.updateAccounts(Map balance)`). +3. Transfer money from one account to another, printing out account balances before and after the transfer (`BasicExampleDAO.transferFunds(int from, int to, int amount)`). +4. Insert random account data using JDBC's bulk insertion support (`BasicExampleDAO.bulkInsertRandomAccountData()`). +5. Print out (some) account data (`BasicExampleDAO.readAccounts(int limit)`). +6. Drop the `accounts` table and perform any other necessary cleanup (`BasicExampleDAO.tearDown()`). (Note that you can run this program as many times as you want due to this cleanup step.) + +It does all of the above using the practices we recommend for using JDBC with CockroachDB, which are listed in the [Recommended Practices](#recommended-practices) section below. + +To run it: + +1. Clone the `hello-world-java-jdbc` repo to your machine: + + {% include copy-clipboard.html %} + ~~~ shell + git clone https://github.com/cockroachlabs/hello-world-java-jdbc/ + ~~~ + +2. Download [the PostgreSQL JDBC driver](https://jdbc.postgresql.org/download.html). +3. Compile and run the code (adding the PostgreSQL JDBC driver to your classpath): + + {% include copy-clipboard.html %} + ~~~ shell + $ javac -classpath .:/path/to/postgresql.jar BasicExample.java + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ java -classpath .:/path/to/postgresql.jar BasicExample + ~~~ + +The contents of [`BasicExample.java`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/v19.1/app/insecure/BasicExample.java): + +{% include copy-clipboard.html %} +~~~ java +{% include {{page.version.version}}/app/insecure/BasicExample.java %} +~~~ + +
+ +The output will look like the following: + +~~~ +BasicExampleDAO.createAccounts: + 'CREATE TABLE IF NOT EXISTS accounts (id INT PRIMARY KEY, balance INT, CONSTRAINT balance_gt_0 CHECK (balance >= 0))' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (1, 1000)' + +BasicExampleDAO.updateAccounts: + 'INSERT INTO accounts (id, balance) VALUES (2, 250)' +BasicExampleDAO.updateAccounts: + => 2 total updated accounts +main: + => Account balances at time '11:54:06.904': + ID 1 => $1000 + ID 2 => $250 + +BasicExampleDAO.transferFunds: + 'UPSERT INTO accounts (id, balance) VALUES(1, ((SELECT balance FROM accounts WHERE id = 1) - 100)),(2, ((SELECT balance FROM accounts WHERE id = 2) + 100))' +BasicExampleDAO.transferFunds: + => $100 transferred between accounts 1 and 2, 2 rows updated +main: + => Account balances at time '11:54:06.985': + ID 1 => $900 + ID 2 => $350 + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (354685257, 158423397)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (206179866, 950590234)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (708995411, 892928833)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + 'INSERT INTO accounts (id, balance) VALUES (500817884, 189050420)' + => 128 row(s) updated in this batch + +BasicExampleDAO.bulkInsertRandomAccountData: + => finished, 512 total rows inserted + +BasicExampleDAO.readAccounts: + 'SELECT id, balance FROM accounts LIMIT 10' + id => 1 + balance => 900 + id => 2 + balance => 350 + id => 190756 + balance => 966414958 + id => 1002343 + balance => 243354081 + id => 1159751 + balance => 59745201 + id => 2193125 + balance => 346719279 + id => 2659707 + balance => 770266587 + id => 6819325 + balance => 511618834 + id => 9985390 + balance => 905049643 + id => 12256472 + balance => 913034434 + +BasicExampleDAO.tearDown: + 'DROP TABLE accounts' +~~~ + +## Recommended Practices + +### Use `IMPORT` to read in large data sets + +If you are trying to get a large data set into CockroachDB all at once (a bulk import), avoid writing client-side code altogether and use the [`IMPORT`](import.html) statement instead. It is much faster and more efficient than making a series of [`INSERT`s](insert.html) and [`UPDATE`s](update.html). It bypasses the [SQL layer](architecture/sql-layer.html) altogether and writes directly to the [storage layer](architecture/storage-layer.html) of the database. + +For more information about importing data from Postgres, see [Migrate from Postgres](migrate-from-postgres.html). + +For more information about importing data from MySQL, see [Migrate from MySQL](migrate-from-mysql.html). + +### Use `rewriteBatchedInserts` for increased speed + +We strongly recommend setting `rewriteBatchedInserts=true`; we have seen 2-3x performance improvements with it enabled. From [the JDBC connection parameters documentation](https://jdbc.postgresql.org/documentation/head/connect.html#connection-parameters): + +> This will change batch inserts from `insert into foo (col1, col2, col3) values (1,2,3)` into `insert into foo (col1, col2, col3) values (1,2,3), (4,5,6)` this provides 2-3x performance improvement + +### Use a batch size of 128 + +PGJDBC's batching support only works with [powers of two](https://github.com/pgjdbc/pgjdbc/blob/7b52b0c9e5b9aa9a9c655bb68f23bf4ec57fd51c/pgjdbc/src/main/java/org/postgresql/jdbc/PgPreparedStatement.java#L1597), and will split batches of other sizes up into multiple sub-batches. This means that a batch of size 128 can be 6x faster than a batch of size 250. + +The code snippet below shows a pattern for using a batch size of 128, and is taken from the longer example above (specifically, the `BasicExampleDAO.bulkInsertRandomAccountData()` method). + +Specifically, it does the following: + +1. Turn off auto-commit so you can manage the transaction lifecycle and thus the size of the batch inserts. +2. Given an overall update size of 500 rows (for example), split it into batches of size 128 and execute each batch in turn. +3. Finally, commit the batches of statements you've just executed. + +~~~ java +int BATCH_SIZE = 128; +connection.setAutoCommit(false); + +try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO accounts (id, balance) VALUES (?, ?)")) { + for (int i=0; i<=(500/BATCH_SIZE);i++) { + for (int j=0; j %s row(s) updated in this batch\n", count.length); // Verifying 128 rows in the batch + } + connection.commit(); +} +~~~ + +### Retrieve large data sets in chunks using cursors + +CockroachDB now supports the Postgres wire-protocol cursors for implicit transactions and explicit transactions executed to completion. This means the [PGJDBC driver](https://jdbc.postgresql.org) can use this protocol to stream queries with large result sets. This is much faster than [paginating through results in SQL using `LIMIT .. OFFSET`](selection-queries.html#paginate-through-limited-results). + +For instructions showing how to use cursors in your Java code, see [Getting results based on a cursor](https://jdbc.postgresql.org/documentation/head/query.html#query-with-cursor) from the PGJDBC documentation. + +Note that interleaved execution (partial execution of multiple statements within the same connection and transaction) is not supported when [`Statement.setFetchSize()`](https://docs.oracle.com/javase/8/docs/api/java/sql/Statement.html#setFetchSize-int-) is used. + +## What's next? + +Read more about using the [Java JDBC driver](https://jdbc.postgresql.org/). + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-nodejs-app-with-cockroachdb-sequelize.md b/v20.2/build-a-nodejs-app-with-cockroachdb-sequelize.md new file mode 100644 index 00000000000..90c21b059f1 --- /dev/null +++ b/v20.2/build-a-nodejs-app-with-cockroachdb-sequelize.md @@ -0,0 +1,165 @@ +--- +title: Build a Node.js App with CockroachDB and Sequelize +summary: Learn how to use CockroachDB from a simple Node.js application with the Sequelize ORM. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Node.js application with CockroachDB and the Sequelize ORM. + +We have tested the [Sequelize ORM](https://sequelize.readthedocs.io/en/v3/) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +{{site.data.alerts.callout_success}} +For a more realistic use of Sequelize with CockroachDB, see our [`examples-orms`](https://github.com/cockroachdb/examples-orms)repository. +{{site.data.alerts.end}} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the Sequelize ORM + +To install Sequelize, as well as a [CockroachDB Node.js package](https://github.com/cockroachdb/sequelize-cockroachdb) that accounts for some minor differences between CockroachDB and PostgreSQL, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ npm install sequelize sequelize-cockroachdb +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Node.js code + +The following code uses the [Sequelize](https://sequelize.readthedocs.io/en/v3/) ORM to map Node.js-specific objects to SQL operations. Specifically, `Account.sync({force: true})` creates an `accounts` table based on the Account model (or drops and recreates the table if it already exists), `Account.bulkCreate([...])` inserts rows into the table, and `Account.findAll()` selects from the table so that balances can be printed. + +Copy the code or +download it directly. + +{% include copy-clipboard.html %} +~~~ js +{% include {{ page.version.version }}/app/sequelize-basic-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node sequelize-basic-sample.js +~~~ + +The output should be: + +~~~ shell +1 1000 +2 250 +~~~ + +To verify that funds were transferred from one account to another, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=/tmp/certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 1000 | +| 2 | 250 | ++----+---------+ +(2 rows) +~~~ + +
+ + + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Node.js code + +The following code uses the [Sequelize](https://sequelize.readthedocs.io/en/v3/) ORM to map Node.js-specific objects to SQL operations. Specifically, `Account.sync({force: true})` creates an `accounts` table based on the Account model (or drops and recreates the table if it already exists), `Account.bulkCreate([...])` inserts rows into the table, and `Account.findAll()` selects from the table so that balances can be printed. + +Copy the code or +download it directly. + +{% include copy-clipboard.html %} +~~~ js +{% include {{ page.version.version }}/app/insecure/sequelize-basic-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node sequelize-basic-sample.js +~~~ + +The output should be: + +~~~ shell +1 1000 +2 250 +~~~ + +To verify that the table and rows were created successfully, you can again use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SHOW TABLES' --database=bank +~~~ + +~~~ ++------------+ +| table_name | ++------------+ +| accounts | ++------------+ +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 1000 | +| 2 | 250 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Sequelize ORM](https://sequelize.readthedocs.io/en/v3/), or check out a more realistic implementation of Sequelize with CockroachDB in our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-nodejs-app-with-cockroachdb.md b/v20.2/build-a-nodejs-app-with-cockroachdb.md new file mode 100644 index 00000000000..7b95ac721a7 --- /dev/null +++ b/v20.2/build-a-nodejs-app-with-cockroachdb.md @@ -0,0 +1,230 @@ +--- +title: Build a Node.js App with CockroachDB and the Node.js pg Driver +summary: Learn how to use CockroachDB from a simple Node.js application with the Node.js pg driver. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Node.js application with CockroachDB and the Node.js pg driver. + +We have tested the [Node.js pg driver](https://www.npmjs.com/package/pg) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install Node.js packages + +To let your application communicate with CockroachDB, install the [Node.js pg driver](https://www.npmjs.com/package/pg): + +{% include copy-clipboard.html %} +~~~ shell +$ npm install pg +~~~ + +The example app on this page also requires [`async`](https://www.npmjs.com/package/async): + +{% include copy-clipboard.html %} +~~~ shell +$ npm install async +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Node.js code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the [`basic-sample.js`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/basic-sample.js) file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ js +{% include {{page.version.version}}/app/basic-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node basic-sample.js +~~~ + +The output should be: + +~~~ +Initial balances: +{ id: '1', balance: '1000' } +{ id: '2', balance: '250' } +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another and then read the updated values, where all included statements are either committed or aborted. + +Download the [`txn-sample.js`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/txn-sample.js) file, or create the file yourself and copy the code into it. + +{% include {{ page.version.version }}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ js +{% include {{page.version.version}}/app/txn-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node txn-sample.js +~~~ + +The output should be: + +~~~ +Balances after transfer: +{ id: '1', balance: '900' } +{ id: '2', balance: '350' } +~~~ + +To verify that funds were transferred from one account to another, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Node.js code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, creating a table, inserting rows, and reading and printing the rows. + +Download the [`basic-sample.js`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/insecure/basic-sample.js) file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ js +{% include {{page.version.version}}/app/insecure/basic-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node basic-sample.js +~~~ + +The output should be: + +~~~ +Initial balances: +{ id: '1', balance: '1000' } +{ id: '2', balance: '250' } +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another and then read the updated values, where all included statements are either committed or aborted. + +Download the [`txn-sample.js`](https://raw.githubusercontent.com/cockroachdb/docs/master/_includes/{{ page.version.version }}/app/insecure/txn-sample.js) file, or create the file yourself and copy the code into it. + +{% include {{ page.version.version }}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ js +{% include {{page.version.version}}/app/insecure/txn-sample.js %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ node txn-sample.js +~~~ + +The output should be: + +~~~ +Balances after transfer: +{ id: '1', balance: '900' } +{ id: '2', balance: '350' } +~~~ + +To verify that funds were transferred from one account to another, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Node.js pg driver](https://www.npmjs.com/package/pg). + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-php-app-with-cockroachdb.md b/v20.2/build-a-php-app-with-cockroachdb.md new file mode 100644 index 00000000000..d42838d8509 --- /dev/null +++ b/v20.2/build-a-php-app-with-cockroachdb.md @@ -0,0 +1,175 @@ +--- +title: Build a PHP App with CockroachDB and php-pgsql +summary: Learn how to use CockroachDB from a simple PHP application with a low-level client driver. +toc: true +twitter: false +--- + +This tutorial shows you how build a simple PHP application with CockroachDB and the php-pgsql driver. + +We have tested the [php-pgsql driver](http://php.net/manual/en/book.pgsql.php) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the php-pgsql driver + +Install the php-pgsql driver as described in the [official documentation](http://php.net/manual/en/book.pgsql.php). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the PHP code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Download the basic-sample.php file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ php +{% include {{ page.version.version }}/app/basic-sample.php %} +~~~ + +The output should be: + +~~~ shell +Account balances: +1: 1000 +2: 250 +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.php file, or create the file yourself and copy the code into it. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ php +{% include {{ page.version.version }}/app/txn-sample.php %} +~~~ + +The output should be: + +~~~ shell +Account balances after transfer: +1: 900 +2: 350 +~~~ + +To verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the PHP code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Download the basic-sample.php file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ php +{% include {{ page.version.version }}/app/insecure/basic-sample.php %} +~~~ + +The output should be: + +~~~ shell +Account balances: +1: 1000 +2: 250 +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.php file, or create the file yourself and copy the code into it. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic **retry function** that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ php +{% include {{ page.version.version }}/app/insecure/txn-sample.php %} +~~~ + +The output should be: + +~~~ shell +Account balances after transfer: +1: 900 +2: 350 +~~~ + +To verify that funds were transferred from one account to another, use the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [php-pgsql driver](http://php.net/manual/en/book.pgsql.php). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-a-python-app-with-cockroachdb-django.md b/v20.2/build-a-python-app-with-cockroachdb-django.md new file mode 100644 index 00000000000..5463379f8f4 --- /dev/null +++ b/v20.2/build-a-python-app-with-cockroachdb-django.md @@ -0,0 +1,471 @@ +--- +title: Build a Python App with CockroachDB and Django +summary: Learn how to use CockroachDB from a simple Django application. +toc: true +twitter: false +build_for: [cockroachdb] +--- + +{% unless site.cockroachcloud %} + + + +{% endunless %} + +This tutorial shows you how build a simple Python application with CockroachDB and the [Django](https://www.djangoproject.com/) framework. + +CockroachDB supports Django versions 2.2 and 3.0. + +{% unless site.cockroachcloud %} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +{% endunless %} + +{{site.data.alerts.callout_info}} +The example code and instructions on this page use Python 3 and Django 3.0. +{{site.data.alerts.end}} + +## Step 1. Install Django and the CockroachDB backend for Django + +Install [Django](https://docs.djangoproject.com/en/3.0/topics/install/) and the [CockroachDB backend for Django](https://github.com/cockroachdb/django-cockroachdb): + +{% include copy-clipboard.html %} +~~~ shell +$ python3 -m pip install django==3.0.* +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ python3 -m pip install django-cockroachdb==3.0.* +~~~ + +{{site.data.alerts.callout_info}} +The major version of `django-cockroachdb` must correspond to the major version of `django`. The minor release numbers do not need to match. +{{site.data.alerts.end}} + +{% unless site.cockroachcloud %} + +
+ +## Step 2. Create the `django` user and `bank` database + +Open a [SQL shell](use-the-built-in-sql-client.html) to the running CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --host=localhost:26257 +~~~ + +In the SQL shell, issue the following statements to create the `django` user and `bank` database: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER IF NOT EXISTS django WITH PASSWORD 'password'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +Give the `django` user the necessary permissions: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE bank TO django; +~~~ + +Exit the SQL shell: + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ + +
+ +
+ +## Step 2. Create the `django` user and `bank` database + +Open a [SQL shell](use-the-built-in-sql-client.html) to the running CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost:26257 +~~~ + +In the SQL shell, issue the following statements to create the `django` user and `bank` database: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER IF NOT EXISTS django; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +Give the `django` user the necessary permissions: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE bank TO django; +~~~ + +Exit the SQL shell: + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ + +
+ +{% endunless %} + +{% if site.cockroachcloud %} + +## Step 2: Connect to your CockroachCloud cluster and create the `django` user and `bank` database + +Connect to your CockroachCloud cluster using the [SQL shell](cockroachcloud-connect-to-your-cluster.html#use-the-cockroachdb-sql-client). + +In the SQL shell, issue the following statements to create the `django` user and `bank` database: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER IF NOT EXISTS django WITH PASSWORD 'password'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +Give the `django` user the necessary permissions: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE bank TO django; +~~~ + +Exit the SQL shell: + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ + +{% endif %} + +## Step 3. Create a Django project + +In the directory where you'd like to store your code, use the [`django-admin` command-line tool](https://docs.djangoproject.com/en/3.0/ref/django-admin/) to create an application project: + +{% include copy-clipboard.html %} +~~~ shell +$ django-admin startproject myproject +~~~ + +This creates a new project directory called `myproject`. `myproject` contains the [`manage.py` script](https://docs.djangoproject.com/en/3.0/ref/django-admin/) and a subdirectory, also named `myproject`, that contains some `.py` files. + +Open `myproject/myproject/settings.py`, and add `0.0.0.0` to the `ALLOWED_HOSTS` in your `settings.py` file, so that it reads as follows: + +{% include copy-clipboard.html %} +~~~ python +ALLOWED_HOSTS = ['0.0.0.0'] +~~~ + +In `myproject/myproject/settings.py`, add `myproject` to the list of `INSTALLED_APPS`, so that it reads as follows: + +{% include copy-clipboard.html %} +~~~ python +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'myproject', +] +~~~ + +The other installed applications listed are added to all starter Django applications by default. + +{% unless site.cockroachcloud %} + +In `myproject/myproject/settings.py`, change `DATABASES` to the following: + +
+ +{% include copy-clipboard.html %} +~~~ python +DATABASES = { + 'default': { + 'ENGINE': 'django_cockroachdb', + 'NAME': 'bank', + 'USER': 'django', + 'PASSWORD': 'password', + 'HOST': 'localhost', + 'PORT': '26257', + } +} +~~~ + +
+ +
+ +{% include copy-clipboard.html %} +~~~ python +DATABASES = { + 'default': { + 'ENGINE': 'django_cockroachdb', + 'NAME': 'bank', + 'USER': 'django', + 'HOST': 'localhost', + 'PORT': '26257', + } +} +~~~ + +
+ +{% endunless %} + +{% if site.cockroachcloud %} + +In the CockroachCloud Console, generate the [connection parameters](cockroachcloud-connect-to-your-cluster.html#step-3-select-a-connection-method). Then in `myproject/myproject/settings.py`, change `DATABASES` to the following: + +{% include copy-clipboard.html %} +~~~ python +DATABASES = { + 'default': { + 'ENGINE': 'django_cockroachdb', + 'NAME': 'bank', + 'USER': 'django', + 'PASSWORD': 'password', + 'HOST': '', + 'PORT': '26257', + } +} +~~~ + +{% endif %} + +## Step 4. Write the application logic + +After you generate the initial Django project files, you need to build out the application with a few `.py` files in `myproject/myproject`. + +
+ +### Models + +Start by building some [models](https://docs.djangoproject.com/en/3.0/topics/db/models/), defined in a file called `models.py`. You can copy the sample code below and paste it into a new file, or you can download the file directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/django-basic-sample/models.py %} +~~~ + +In this file, we define some simple classes that map to the tables in the example database `bank`. + +### Views + +Next, build out some [class-based views](https://docs.djangoproject.com/en/3.0/topics/class-based-views/) for the application in a file called `views.py`. You can copy the sample code below and paste it into a new file, or you can download the file directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/django-basic-sample/views.py %} +~~~ + +This file defines the application's views as classes. Each view class corresponds to one of the table classes defined in `models.py`. The methods of these classes define read and write transactions on the tables in the database. + +Importantly, the file defines a [transaction retry loop](transactions.html#transaction-retries) in the decorator function `retry_on_exception()`. This function decorates each view method, ensuring that transaction ordering guarantees meet the ANSI [SERIALIZABLE](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) isolation level. For more information about how transactions (and retries) work, see [Transactions](transactions.html). + +### URL routes + +Lastly, define some [URL routes](https://docs.djangoproject.com/en/3.0/topics/http/urls/) in a file called `urls.py`. The `django-admin` command-line tool generated this file when you created the Django project, so it should already exist in `myproject/myproject`. You can copy the sample code below and paste it into the existing `urls.py` file, or you can download the file directly and replace the existing one. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/django-basic-sample/urls.py %} +~~~ + +
+ +
+ +### Models + +Start by building some [models](https://docs.djangoproject.com/en/3.0/topics/db/models/), defined in a file called `models.py`. You can copy the sample code below and paste it into a new file, or you can download the file directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/insecure/django-basic-sample/models.py %} +~~~ + +In this file, we define some simple classes that map to the tables in the example database `bank`. + +### Views + +Next, build out some [class-based views](https://docs.djangoproject.com/en/3.0/topics/class-based-views/) for the application in a file called `views.py`. You can copy the sample code below and paste it into a new file, or you can download the file directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/insecure/django-basic-sample/views.py %} +~~~ + +This file defines the application's views as classes. Each view class corresponds to one of the table classes defined in `models.py`. The methods of these classes define read and write transactions on the tables in the database. + +Importantly, the file defines a [transaction retry loop](transactions.html#transaction-retries) in the decorator function `retry_on_exception()`. This function decorates each view method, ensuring that transaction ordering guarantees meet the ANSI [SERIALIZABLE](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) isolation level. For more information about how transactions (and retries) work, see [Transactions](transactions.html). + +### URL routes + +Lastly, define some [URL routes](https://docs.djangoproject.com/en/3.0/topics/http/urls/) in a file called `urls.py`. The `django-admin` command-line tool generated this file when you created the Django project, so it should already exist in `myproject/myproject`. You can copy the sample code below and paste it into the existing `urls.py` file, or you can download the file directly and replace the existing one. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/insecure/django-basic-sample/urls.py %} +~~~ + +
+ +## Step 5. Set up and run the Django app + +In the top `myproject` directory, use the [`manage.py` script](https://docs.djangoproject.com/en/3.0/ref/django-admin/) to create [Django migrations](https://docs.djangoproject.com/en/3.0/topics/migrations/) that initialize the database for the application: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 manage.py makemigrations myproject +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ python3 manage.py migrate +~~~ + +This initializes the `bank` database with the tables defined in `models.py`, in addition to some other tables for the admin functionality included with Django's starter application. + +{% unless site.cockroachcloud %} + +
+ +To verify that the migration succeeded, open a [SQL shell](use-the-built-in-sql-client.html) to the running CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --host=localhost:26257 +~~~ + +
+ +
+ +To verify that the migration succeeded, open a [SQL shell](use-the-built-in-sql-client.html) to the running CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost:26257 +~~~ + +
+ +{% endunless %} + +{% if site.cockroachcloud %} + +To verify that the migration succeeded, connect to your CockroachCloud cluster using the [SQL shell](cockroachcloud-connect-to-your-cluster.html#use-the-cockroachdb-sql-client) and issue the following statements: + +{% endif %} + +{% include copy-clipboard.html %} +~~~ sql +> USE bank; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES; +~~~ + +~~~ + table_name ++----------------------------+ + auth_group + auth_group_permissions + auth_permission + auth_user + auth_user_groups + auth_user_user_permissions + django_admin_log + django_content_type + django_migrations + django_session + myproject_customers + myproject_orders + myproject_orders_product + myproject_products +(14 rows) +~~~ + +In a new terminal, start the app: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 manage.py runserver 0.0.0.0:8000 +~~~ + +To perform simple reads and writes to the database, you can send HTTP requests to the application. + +For example, in a new terminal, you can use `curl` to send a POST request to the application that inserts a new row into the `customers` table: + +{% include copy-clipboard.html %} +~~~ shell +$ curl --header "Content-Type: application/json" \ +--request POST \ +--data '{"name":"Carl"}' http://0.0.0.0:8000/customer/ +~~~ + +You can then send a GET request to read from that table: + +{% include copy-clipboard.html %} +~~~ shell +$ curl http://0.0.0.0:8000/customer/ +~~~ + +~~~ +[{"id": 523377322022797313, "name": "Carl"}] +~~~ + +You can also query the tables directly in the SQL shell to see the changes: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM myproject_customers; +~~~ + +~~~ + id | name ++--------------------+------+ + 523377322022797313 | Carl +(1 row) +~~~ + + +## What's next? + +Read more about writing a [Django app](https://docs.djangoproject.com/en/3.0/intro/tutorial01/). + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-python-app-with-cockroachdb-pony.md b/v20.2/build-a-python-app-with-cockroachdb-pony.md new file mode 100644 index 00000000000..27fb625efc1 --- /dev/null +++ b/v20.2/build-a-python-app-with-cockroachdb-pony.md @@ -0,0 +1,139 @@ +--- +title: Build a Python App with CockroachDB and PonyORM +summary: Learn how to use CockroachDB from a simple Python application with PonyORM. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Python application with CockroachDB and [PonyORM](https://ponyorm.org/). + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +{{site.data.alerts.callout_info}} +The example code on this page uses Python 3. +{{site.data.alerts.end}} + + +## Step 1. Install PonyORM + +To install PonyORM run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 -m pip install pony +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a client certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Python code + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Python code + +
+ +The code below uses PonyORM to map Python objects and methods to SQL operations. When you run the code as a script, it performs the following operations: + +1. Reads existing account IDs from the `bank` database. +2. Creates additional accounts with randomly generated IDs, and then adds a bit of money to each new account. +3. Chooses two accounts at random and takes half of the money from the first account and deposits it into the second. + +
+ +Copy the code below to a file or +download it directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/pony-basic-sample.py %} +~~~ + +
+ +
+ +Copy the code below to a file or +download it directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/insecure/pony-basic-sample.py %} +~~~ + +
+ +Run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 pony-basic-sample.py +~~~ + +To verify that the table and rows were created successfully, open a new terminal, and start a new session with the built-in SQL client: + +
+ +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank +~~~ + +
+ +
+ +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +
+ +Issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT COUNT(*) FROM accounts; +~~~ + +~~~ + count +------- + 100 +(1 row) +~~~ + +## Best practices + +Pony ORM provides the [retry option](transactions.html#client-side-intervention) for the `db_session` decorator. If Pony detects that the optimistic checks do not pass, it restarts the decorated function automatically. +The `retry` parameter can only be specified in the `db_session` decorator and not the context manager. For more information, see [PonyORM documentation](https://docs.ponyorm.org/api_reference.html?highlight=retry#transactions-db-session). diff --git a/v20.2/build-a-python-app-with-cockroachdb-sqlalchemy.md b/v20.2/build-a-python-app-with-cockroachdb-sqlalchemy.md new file mode 100644 index 00000000000..61666bcf81d --- /dev/null +++ b/v20.2/build-a-python-app-with-cockroachdb-sqlalchemy.md @@ -0,0 +1,331 @@ +--- +title: Build a Python App with CockroachDB and SQLAlchemy +summary: Learn how to use CockroachDB from a simple Python application with SQLAlchemy. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Python application with CockroachDB and the [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) ORM. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +{{site.data.alerts.callout_info}} +The example code on this page uses Python 3. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_danger}} +SQLAlchemy relies on the existence of [foreign keys](foreign-key.html) to generate [`JOIN` expressions](joins.html) from your application code. If you remove foreign keys from your schema, SQLAlchemy won't generate joins for you. As a workaround, you can [create a "custom foreign condition" by adding a `relationship` field to your table objects](https://stackoverflow.com/questions/37806625/sqlalchemy-create-relations-but-without-foreign-key-constraint-in-db), or do the equivalent work in your application. +{{site.data.alerts.end}} + +## Step 1. Install SQLAlchemy + +To install SQLAlchemy, as well as a [CockroachDB Python package](https://github.com/cockroachdb/sqlalchemy-cockroachdb) that accounts for some differences between CockroachDB and PostgreSQL, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ pip install sqlalchemy sqlalchemy-cockroachdb psycopg2 +~~~ + +{{site.data.alerts.callout_success}} +You can substitute psycopg2 for other alternatives that include the psycopg python package. +{{site.data.alerts.end}} + +For other ways to install SQLAlchemy, see the [official documentation](http://docs.sqlalchemy.org/en/latest/intro.html#installation-guide). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Python code + +The code below uses [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) to map Python objects and methods to SQL operations. + +You can run this script as many times as you want; on each run, the script will create some new accounts and shuffle money around between randomly selected accounts. + +Specifically, the script: + +1. Reads in existing account IDs (if any) from the `bank` database. +2. Creates additional accounts with randomly generated IDs. Then, it adds a bit of money to each new account. +3. Chooses two accounts at random and takes half of the money from the first and deposits it into the second. + +It does all of the above using the practices we recommend for using SQLAlchemy with CockroachDB, which are listed in the [Best practices](#best-practices) section below. + +{{site.data.alerts.callout_info}} +You must use the `cockroachdb://` prefix in the URL passed to [`sqlalchemy.create_engine`](https://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#sqlalchemy.create_engine) to make sure the [`cockroachdb`](https://github.com/cockroachdb/sqlalchemy-cockroachdb) dialect is used. Using the `postgres://` URL prefix to connect to your CockroachDB cluster will not work. +{{site.data.alerts.end}} + +Copy the code below or +download it directly. + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-python-sqlalchemy/` +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/sqlalchemy-basic-sample.py %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 sqlalchemy-basic-sample.py +~~~ + +The output should look something like the following: + +~~~ shell +2018-12-06 15:59:58,999 INFO sqlalchemy.engine.base.Engine select current_schema() +2018-12-06 15:59:58,999 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1 +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1 +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,002 INFO sqlalchemy.engine.base.Engine select version() +2018-12-06 15:59:59,002 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,003 INFO sqlalchemy.engine.base.Engine SELECT table_name FROM information_schema.tables WHERE table_schema=%s +2018-12-06 15:59:59,004 INFO sqlalchemy.engine.base.Engine ('public',) +2018-12-06 15:59:59,005 INFO sqlalchemy.engine.base.Engine SELECT id from accounts; +2018-12-06 15:59:59,005 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine BEGIN (implicit) +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine SAVEPOINT cockroach_restart +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,083 INFO sqlalchemy.engine.base.Engine INSERT INTO accounts (id, balance) VALUES (%(id)s, %(balance)s) +2018-12-06 15:59:59,083 INFO sqlalchemy.engine.base.Engine ({'id': 298865, 'balance': 208217}, {'id': 506738, 'balance': 962549}, {'id': 514698, 'balance': 986327}, {'id': 587747, 'balance': 210406}, {'id': 50148, 'balance': 347976}, {'id': 854295, 'balance': 420086}, {'id': 785757, 'balance': 364836}, {'id': 406247, 'balance': 787016} ... displaying 10 of 100 total bound parameter sets ... {'id': 591336, 'balance': 542066}, {'id': 33728, 'balance': 526531}) +2018-12-06 15:59:59,201 INFO sqlalchemy.engine.base.Engine RELEASE SAVEPOINT cockroach_restart +2018-12-06 15:59:59,201 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,205 INFO sqlalchemy.engine.base.Engine COMMIT +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine BEGIN (implicit) +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine SAVEPOINT cockroach_restart +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,207 INFO sqlalchemy.engine.base.Engine SELECT accounts.id AS accounts_id, accounts.balance AS accounts_balance +FROM accounts +WHERE accounts.id = %(id_1)s +2018-12-06 15:59:59,207 INFO sqlalchemy.engine.base.Engine {'id_1': 769626} +2018-12-06 15:59:59,209 INFO sqlalchemy.engine.base.Engine UPDATE accounts SET balance=%(balance)s WHERE accounts.id = %(accounts_id)s +2018-12-06 15:59:59,209 INFO sqlalchemy.engine.base.Engine {'balance': 470580, 'accounts_id': 769626} +2018-12-06 15:59:59,212 INFO sqlalchemy.engine.base.Engine UPDATE accounts SET balance=(accounts.balance + %(balance_1)s) WHERE accounts.id = %(id_1)s +2018-12-06 15:59:59,247 INFO sqlalchemy.engine.base.Engine {'balance_1': 470580, 'id_1': 158447} +2018-12-06 15:59:59,249 INFO sqlalchemy.engine.base.Engine RELEASE SAVEPOINT cockroach_restart +2018-12-06 15:59:59,250 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,251 INFO sqlalchemy.engine.base.Engine COMMIT +~~~ + +To verify that the table and rows were created successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank +~~~ + +Then, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT COUNT(*) FROM accounts; +~~~ + +~~~ + count +------- + 100 +(1 row) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Python code + +The code below uses [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) to map Python objects and methods to SQL operations. + +You can run this script as many times as you want; on each run, it will create some new accounts and shuffle money around between randomly selected accounts. + +Specifically, it: + +1. Reads in existing account IDs (if any) from the `bank` database. +2. Creates additional accounts with randomly generated IDs. Then, it adds a bit of money to each new account. +3. Chooses two accounts at random and takes half of the money from the first and deposits it into the second. + +It does all of the above using the practices we recommend for using SQLAlchemy with CockroachDB, which are listed in the [Best practices](#best-practices) section below. + +{{site.data.alerts.callout_info}} +You must use the `cockroachdb://` prefix in the URL passed to [`sqlalchemy.create_engine`](https://docs.sqlalchemy.org/en/latest/core/engines.html?highlight=create_engine#sqlalchemy.create_engine) to make sure the [`cockroachdb`](https://github.com/cockroachdb/sqlalchemy-cockroachdb) dialect is used. Using the `postgres://` URL prefix to connect to your CockroachDB cluster will not work. +{{site.data.alerts.end}} + +To get the code below, clone the `hello-world-python-sqlalchemy` repo to your machine: + +{% include copy-clipboard.html %} +~~~ shell +git clone https://github.com/cockroachlabs/hello-world-python-sqlalchemy/ +~~~ + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/sqlalchemy-basic-sample.py %} +~~~ + +Change to the directory where you cloned the repo and run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ python3 example.py +~~~ + +The output should look something like the following: + +~~~ shell +2018-12-06 15:59:58,999 INFO sqlalchemy.engine.base.Engine select current_schema() +2018-12-06 15:59:58,999 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1 +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1 +2018-12-06 15:59:59,001 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,002 INFO sqlalchemy.engine.base.Engine select version() +2018-12-06 15:59:59,002 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,003 INFO sqlalchemy.engine.base.Engine SELECT table_name FROM information_schema.tables WHERE table_schema=%s +2018-12-06 15:59:59,004 INFO sqlalchemy.engine.base.Engine ('public',) +2018-12-06 15:59:59,005 INFO sqlalchemy.engine.base.Engine SELECT id from accounts; +2018-12-06 15:59:59,005 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine BEGIN (implicit) +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine SAVEPOINT cockroach_restart +2018-12-06 15:59:59,008 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,083 INFO sqlalchemy.engine.base.Engine INSERT INTO accounts (id, balance) VALUES (%(id)s, %(balance)s) +2018-12-06 15:59:59,083 INFO sqlalchemy.engine.base.Engine ({'id': 298865, 'balance': 208217}, {'id': 506738, 'balance': 962549}, {'id': 514698, 'balance': 986327}, {'id': 587747, 'balance': 210406}, {'id': 50148, 'balance': 347976}, {'id': 854295, 'balance': 420086}, {'id': 785757, 'balance': 364836}, {'id': 406247, 'balance': 787016} ... displaying 10 of 100 total bound parameter sets ... {'id': 591336, 'balance': 542066}, {'id': 33728, 'balance': 526531}) +2018-12-06 15:59:59,201 INFO sqlalchemy.engine.base.Engine RELEASE SAVEPOINT cockroach_restart +2018-12-06 15:59:59,201 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,205 INFO sqlalchemy.engine.base.Engine COMMIT +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine BEGIN (implicit) +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine SAVEPOINT cockroach_restart +2018-12-06 15:59:59,206 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,207 INFO sqlalchemy.engine.base.Engine SELECT accounts.id AS accounts_id, accounts.balance AS accounts_balance +FROM accounts +WHERE accounts.id = %(id_1)s +2018-12-06 15:59:59,207 INFO sqlalchemy.engine.base.Engine {'id_1': 769626} +2018-12-06 15:59:59,209 INFO sqlalchemy.engine.base.Engine UPDATE accounts SET balance=%(balance)s WHERE accounts.id = %(accounts_id)s +2018-12-06 15:59:59,209 INFO sqlalchemy.engine.base.Engine {'balance': 470580, 'accounts_id': 769626} +2018-12-06 15:59:59,212 INFO sqlalchemy.engine.base.Engine UPDATE accounts SET balance=(accounts.balance + %(balance_1)s) WHERE accounts.id = %(id_1)s +2018-12-06 15:59:59,247 INFO sqlalchemy.engine.base.Engine {'balance_1': 470580, 'id_1': 158447} +2018-12-06 15:59:59,249 INFO sqlalchemy.engine.base.Engine RELEASE SAVEPOINT cockroach_restart +2018-12-06 15:59:59,250 INFO sqlalchemy.engine.base.Engine {} +2018-12-06 15:59:59,251 INFO sqlalchemy.engine.base.Engine COMMIT +~~~ + +To verify that the table and rows were created successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +Then, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT COUNT(*) FROM accounts; +~~~ + +~~~ + count +------- + 100 +(1 row) +~~~ + +
+ +## Best practices + +### Use the `run_transaction` function + +We strongly recommend using the [`cockroachdb.sqlalchemy.run_transaction()`](https://github.com/cockroachdb/sqlalchemy-cockroachdb/blob/master/cockroachdb/sqlalchemy/transaction.py) function as shown in the code samples on this page. This abstracts the details of [transaction retries](transactions.html#transaction-retries) away from your application code. Transaction retries are more frequent in CockroachDB than in some other databases because we use [optimistic concurrency control](https://en.wikipedia.org/wiki/Optimistic_concurrency_control) rather than locking. Because of this, a CockroachDB transaction may have to be tried more than once before it can commit. This is part of how we ensure that our transaction ordering guarantees meet the ANSI [SERIALIZABLE](https://en.wikipedia.org/wiki/Isolation_(database_systems)#Serializable) isolation level. + +In addition to the above, using `run_transaction` has the following benefits: + +- Because it must be passed a [sqlalchemy.orm.session.sessionmaker](https://docs.sqlalchemy.org/en/latest/orm/session_api.html#session-and-sessionmaker) object (*not* a [session][session]), it ensures that a new session is created exclusively for use by the callback, which protects you from accidentally reusing objects via any sessions created outside the transaction. +- It abstracts away the [client-side transaction retry logic](transactions.html#client-side-intervention) from your application, which keeps your application code portable across different databases. For example, the sample code given on this page works identically when run against Postgres (modulo changes to the prefix and port number in the connection string). +- It will run your transactions to completion much faster than a naive implementation that created a fresh transaction after every retry error. Because of the way the CockroachDB dialect's driver structures the transaction attempts (using a [`SAVEPOINT`](savepoint.html) statement under the hood, which has a slightly different meaning in CockroachDB than in some other databases), the server is able to preserve some information about the previously attempted transactions to allow subsequent retries to complete more easily. + +For more information about how transactions (and retries) work, see [Transactions](transactions.html). + +### Avoid mutations of session and/or transaction state inside `run_transaction()` + +In general, this is in line with the recommendations of the [SQLAlchemy FAQs](https://docs.sqlalchemy.org/en/latest/orm/session_basics.html#session-frequently-asked-questions), which state (with emphasis added by the original author) that + +> As a general rule, the application should manage the lifecycle of the session *externally* to functions that deal with specific data. This is a fundamental separation of concerns which keeps data-specific operations agnostic of the context in which they access and manipulate that data. + +and + +> Keep the lifecycle of the session (and usually the transaction) **separate and external**. + +In keeping with the above recommendations from the official docs, we **strongly recommend** avoiding any explicit mutations of the transaction state inside the callback passed to `run_transaction`, since that will lead to breakage. Specifically, do not make calls to the following functions from inside `run_transaction`: + +- [`sqlalchemy.orm.Session.commit()`](https://docs.sqlalchemy.org/en/latest/orm/session_api.html?highlight=commit#sqlalchemy.orm.session.Session.commit) (or other variants of `commit()`): This is not necessary because `cockroachdb.sqlalchemy.run_transaction` handles the savepoint/commit logic for you. +- [`sqlalchemy.orm.Session.rollback()`](https://docs.sqlalchemy.org/en/latest/orm/session_api.html?highlight=rollback#sqlalchemy.orm.session.Session.rollback) (or other variants of `rollback()`): This is not necessary because `cockroachdb.sqlalchemy.run_transaction` handles the commit/rollback logic for you. +- [`Session.flush()`][session.flush]: This will not work as expected with CockroachDB because CockroachDB does not support nested transactions, which are necessary for `Session.flush()` to work properly. If the call to `Session.flush()` encounters an error and aborts, it will try to rollback. This will not be allowed by the currently-executing CockroachDB transaction created by `run_transaction()`, and will result in an error message like the following: `sqlalchemy.orm.exc.DetachedInstanceError: Instance is not bound to a Session; attribute refresh operation cannot proceed (Background on this error at: http://sqlalche.me/e/bhk3)`. + +### Break up large transactions into smaller units of work + +If you see an error message like `transaction is too large to complete; try splitting into pieces`, you are trying to commit too much data in a single transaction. As described in our [Cluster Settings](cluster-settings.html) docs, the size limit for transactions is defined by the `kv.transaction.max_intents_bytes` setting, which defaults to 256 KiB. Although this setting can be changed by an admin, we strongly recommend against it in most cases. + +Instead, we recommend breaking your transaction into smaller units of work (or "chunks"). A pattern that works for inserting large numbers of objects using `run_transaction` to handle retries automatically for you is shown below. + +~~~ python +{% include {{page.version.version}}/app/sqlalchemy-large-txns.py %} +~~~ + +### Use `IMPORT` to read in large data sets + +If you are trying to get a large data set into CockroachDB all at once (a bulk import), avoid writing client-side code that uses an ORM and use the [`IMPORT`](import.html) statement instead. It is much faster and more efficient than making a series of [`INSERT`s](insert.html) and [`UPDATE`s](update.html) such as are generated by calls to [`session.bulk_save_objects()`](https://docs.sqlalchemy.org/en/latest/orm/session_api.html?highlight=bulk_save_object#sqlalchemy.orm.session.Session.bulk_save_objects). + +For more information about importing data from Postgres, see [Migrate from Postgres](migrate-from-postgres.html). + +For more information about importing data from MySQL, see [Migrate from MySQL](migrate-from-mysql.html). + +### Prefer the query builder + +In general, we recommend using the query-builder APIs of SQLAlchemy (e.g., [`Engine.execute()`](https://docs.sqlalchemy.org/en/latest/core/connections.html?highlight=execute#sqlalchemy.engine.Engine.execute)) in your application over the [Session][session]/ORM APIs if at all possible. That way, you know exactly what SQL is being generated and sent to CockroachDB, which has the following benefits: + +- It's easier to debug your SQL queries and make sure they are working as expected. +- You can more easily tune SQL query performance by issuing different statements, creating and/or using different indexes, etc. For more information, see [SQL Performance Best Practices](performance-best-practices-overview.html). + +## See also + +- The [SQLAlchemy](https://docs.sqlalchemy.org/en/latest/) docs +- [Transactions](transactions.html) + +{% include {{page.version.version}}/app/see-also-links.md %} + + + +[session.flush]: https://docs.sqlalchemy.org/en/latest/orm/session_api.html#sqlalchemy.orm.session.Session.flush +[session]: https://docs.sqlalchemy.org/en/latest/orm/session.html diff --git a/v20.2/build-a-python-app-with-cockroachdb.md b/v20.2/build-a-python-app-with-cockroachdb.md new file mode 100644 index 00000000000..181c985a690 --- /dev/null +++ b/v20.2/build-a-python-app-with-cockroachdb.md @@ -0,0 +1,139 @@ +--- +title: Build a Python App with CockroachDB and psycopg2 +summary: Learn how to use CockroachDB from a simple Python application with the psycopg2 driver. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Python application with CockroachDB and the psycopg2 driver. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the psycopg2 driver + +To install the Python psycopg2 driver, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ pip install psycopg2 +~~~ + +For other ways to install psycopg2, see the [official documentation](http://initd.org/psycopg/docs/install.html). + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Python code + +Now that you have a database and a user, you'll run the code shown below to: + +- Create an `accounts` table and insert some rows. +- Transfer funds between two accounts inside a [transaction](transactions.html). To ensure that we [handle transaction retry errors](transactions.html#client-side-intervention), we write an application-level retry loop that, in case of error, sleeps before trying the funds transfer again. If it encounters another retry error, it sleeps for a longer interval, implementing [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff). +- Finally, we delete the accounts from the table before exiting so we can re-run the example code. + +{{site.data.alerts.callout_success}} +To clone a version of the code below that connects to insecure clusters, run the command below. Note that you will need to edit the connection string to use the certificates that you generated when you set up your secure cluster. + +`git clone https://github.com/cockroachlabs/hello-world-python-psycopg2/` +{{site.data.alerts.end}} + +Copy the code or download it directly. + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/basic-sample.py %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ python basic-sample.py +~~~ + +The output should show the account balances before and after the funds transfer: + +~~~ +Balances at Wed Aug 7 12:11:23 2019 +['1', '1000'] +['2', '250'] +Balances at Wed Aug 7 12:11:23 2019 +['1', '900'] +['2', '350'] +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Python code + +Now that you have a database and a user, you'll run the code shown below to: + +- Create an `accounts` table and insert some rows. +- Transfer funds between two accounts inside a [transaction](transactions.html). To ensure that we [handle transaction retry errors](transactions.html#client-side-intervention), we write an application-level retry loop that, in case of error, sleeps before trying the funds transfer again. If it encounters another retry error, it sleeps for a longer interval, implementing [exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff). +- Finally, we delete the accounts from the table before exiting so we can re-run the example code. + +To get the code below, clone the `hello-world-python-psycopg2` repo to your machine: + +{% include copy-clipboard.html %} +~~~ shell +git clone https://github.com/cockroachlabs/hello-world-python-psycopg2/ +~~~ + +{% include copy-clipboard.html %} +~~~ python +{% include {{page.version.version}}/app/insecure/basic-sample.py %} +~~~ + +Change to the directory where you cloned the repo and run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ python example.py +~~~ + +The output should show the account balances before and after the funds transfer: + +~~~ +Balances at Wed Jul 24 15:58:40 2019 +['1', '1000'] +['2', '250'] +Balances at Wed Jul 24 15:58:40 2019 +['1', '900'] +['2', '350'] +~~~ + +
+ +## What's next? + +Read more about using the [Python psycopg2 driver](http://initd.org/psycopg/docs/). + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-ruby-app-with-cockroachdb-activerecord.md b/v20.2/build-a-ruby-app-with-cockroachdb-activerecord.md new file mode 100644 index 00000000000..68d5d001207 --- /dev/null +++ b/v20.2/build-a-ruby-app-with-cockroachdb-activerecord.md @@ -0,0 +1,171 @@ +--- +title: Build a Ruby App with CockroachDB and ActiveRecord +summary: Learn how to use CockroachDB from a simple Ruby application with the ActiveRecord ORM. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Ruby application with CockroachDB and the ActiveRecord ORM. + +We have tested the [ActiveRecord ORM](http://guides.rubyonrails.org/active_record_basics.html) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +{{site.data.alerts.callout_success}} +For a more realistic use of ActiveRecord with CockroachDB, see our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. +{{site.data.alerts.end}} + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the ActiveRecord ORM + +To install ActiveRecord as well as the [pg driver](https://rubygems.org/gems/pg) and a [CockroachDB Ruby package](https://github.com/cockroachdb/activerecord-cockroachdb-adapter) that accounts for some minor differences between CockroachDB and PostgreSQL, run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ gem install activerecord pg activerecord-cockroachdb-adapter +~~~ + +{{site.data.alerts.callout_info}} +The exact command above will vary depending on the desired version of ActiveRecord. Specifically, version 4.2.x of ActiveRecord requires version 0.1.x of the adapter; version 5.1.x of ActiveRecord requires version 0.2.x of the adapter; version 5.2.x of ActiveRecord requires version 0.3.0.beta1 of the adapter. +{{site.data.alerts.end}} + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Ruby code + +The following code uses the [ActiveRecord](http://guides.rubyonrails.org/active_record_basics.html) ORM to map Ruby-specific objects to SQL operations. Specifically, `Schema.new.change()` creates an `accounts` table based on the Account model (or drops and recreates the table if it already exists), `Account.create()` inserts rows into the table, and `Account.all` selects from the table so that balances can be printed. + +Copy the code or +download it directly. + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/activerecord-basic-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby activerecord-basic-sample.rb +~~~ + +The output should be: + +~~~ shell +-- create_table(:accounts, {:force=>true}) + -> 0.0361s +1 1000 +2 250 +~~~ + +To verify that the table and rows were created successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank +~~~ + +Then, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 1000 | +| 2 | 250 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Ruby code + +The following code uses the [ActiveRecord](http://guides.rubyonrails.org/active_record_basics.html) ORM to map Ruby-specific objects to SQL operations. Specifically, `Schema.new.change()` creates an `accounts` table based on the Account model (or drops and recreates the table if it already exists), `Account.create()` inserts rows into the table, and `Account.all` selects from the table so that balances can be printed. + +Copy the code or +download it directly. + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/insecure/activerecord-basic-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby activerecord-basic-sample.rb +~~~ + +The output should be: + +~~~ shell +-- create_table(:accounts, {:force=>true}) + -> 0.0361s +1 1000 +2 250 +~~~ + +To verify that the table and rows were created successfully, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +Then, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 1000 | +| 2 | 250 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [ActiveRecord ORM](http://guides.rubyonrails.org/active_record_basics.html), or check out a more realistic implementation of ActiveRecord with CockroachDB in our [`examples-orms`](https://github.com/cockroachdb/examples-orms) repository. + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-ruby-app-with-cockroachdb.md b/v20.2/build-a-ruby-app-with-cockroachdb.md new file mode 100644 index 00000000000..bc7c5c2237c --- /dev/null +++ b/v20.2/build-a-ruby-app-with-cockroachdb.md @@ -0,0 +1,207 @@ +--- +title: Build a Ruby App with CockroachDB and the Ruby pg Driver +summary: Learn how to use CockroachDB from a simple Ruby application with the pg client driver. +toc: true +twitter: false +--- + + + +This tutorial shows you how build a simple Ruby application with CockroachDB and the Ruby pg driver. + +We have tested the [Ruby pg driver](https://rubygems.org/gems/pg) enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +## Step 1. Install the Ruby pg driver + +To install the [Ruby pg driver](https://rubygems.org/gems/pg), run the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ gem install pg +~~~ + +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Ruby code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +The following code connects as the `maxroach` user and executes some basic SQL statements: creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.rb file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/basic-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby basic-sample.rb +~~~ + +The output should be: + +~~~ +Initial balances: +{"id"=>"1", "balance"=>"1000"} +{"id"=>"2", "balance"=>"250"} +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.rb file, or create the file yourself and copy the code into it. + +{% include {{ page.version.version }}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/txn-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby txn-sample.rb +~~~ + +To verify that funds were transferred from one account to another, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --database=bank +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 2. Create the `maxroach` user and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Run the Ruby code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +The following code connects as the `maxroach` user and executes some basic SQL statements: creating a table, inserting rows, and reading and printing the rows. + +Download the basic-sample.rb file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/insecure/basic-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby basic-sample.rb +~~~ + +The output should be: + +~~~ +Initial balances: +{"id"=>"1", "balance"=>"1000"} +{"id"=>"2", "balance"=>"250"} +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.rb file, or create the file yourself and copy the code into it. + +{% include {{ page.version.version }}/client-transaction-retry.md %} + +{% include copy-clipboard.html %} +~~~ ruby +{% include {{page.version.version}}/app/insecure/txn-sample.rb %} +~~~ + +Then run the code: + +{% include copy-clipboard.html %} +~~~ shell +$ ruby txn-sample.rb +~~~ + +To verify that funds were transferred from one account to another, start the [built-in SQL client](cockroach-sql.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=bank +~~~ + +To check the account balances, issue the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id, balance FROM accounts; +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Ruby pg driver](https://rubygems.org/gems/pg). + +{% include {{page.version.version}}/app/see-also-links.md %} diff --git a/v20.2/build-a-rust-app-with-cockroachdb.md b/v20.2/build-a-rust-app-with-cockroachdb.md new file mode 100644 index 00000000000..4bec86afc22 --- /dev/null +++ b/v20.2/build-a-rust-app-with-cockroachdb.md @@ -0,0 +1,161 @@ +--- +title: Build a Rust App with CockroachDB and the Rust Postgres Driver +summary: Learn how to use CockroachDB from a simple Rust application with a low-level client driver. +toc: true +twitter: false +--- + +This tutorial shows you how build a simple Rust application with CockroachDB and the Rust Postgres driver. + +We have tested the Rust Postgres driver enough to claim **beta-level** support. If you encounter problems, please [open an issue](https://github.com/cockroachdb/cockroach/issues/new) with details to help us make progress toward full support. + +## Before you begin + +{% include {{page.version.version}}/app/before-you-begin.md %} + +
+ +## Step 1. Specify the Rust Postgres driver as a dependency + +Update your `Cargo.toml` file to specify a dependency on the Rust Postgres driver, as described in the official documentation. + +Additionally, include the OpenSSL bindings and Rust Postgres OpenSSL crates as dependencies. + +## Step 2. Create the `maxroach` users and `bank` database + +{% include {{page.version.version}}/app/create-maxroach-user-and-bank-database.md %} + +## Step 3. Generate a certificate for the `maxroach` user + +Create a certificate and key for the `maxroach` user by running the following command. The code samples will run as this user. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client maxroach --certs-dir=certs --ca-key=my-safe-directory/ca.key +~~~ + +## Step 4. Run the Rust code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Download the basic-sample.rs file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ rust +{% include {{ page.version.version }}/app/basic-sample.rs %} +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.rs file, or create the file yourself and copy the code into it. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic retry function that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ rust +{% include {{ page.version.version }}/app/txn-sample.rs %} +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +
+ +## Step 1. Specify the Rust Postgres driver as a dependency + +Update your `Cargo.toml` file to specify a dependency on the Rust Postgres driver, as described in the official documentation. + +## Step 2. Create the `maxroach` users and `bank` database + +{% include {{page.version.version}}/app/insecure/create-maxroach-user-and-bank-database.md %} + +## Step 3. Create a table in the new database + +As the `maxroach` user, use the [built-in SQL client](cockroach-sql.html) to create an `accounts` table in the new database. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--database=bank \ +--user=maxroach \ +-e 'CREATE TABLE accounts (id INT PRIMARY KEY, balance INT)' +~~~ + +## Step 4. Run the Rust code + +Now that you have a database and a user, you'll run code to create a table and insert some rows, and then you'll run code to read and update values as an atomic [transaction](transactions.html). + +### Basic statements + +First, use the following code to connect as the `maxroach` user and execute some basic SQL statements, inserting rows and reading and printing the rows. + +Download the basic-sample.rs file, or create the file yourself and copy the code into it. + +{% include copy-clipboard.html %} +~~~ rust +{% include {{ page.version.version }}/app/insecure/basic-sample.rs %} +~~~ + +### Transaction (with retry logic) + +Next, use the following code to again connect as the `maxroach` user but this time execute a batch of statements as an atomic transaction to transfer funds from one account to another, where all included statements are either committed or aborted. + +Download the txn-sample.rs file, or create the file yourself and copy the code into it. + +{{site.data.alerts.callout_info}} +CockroachDB may require the [client to retry a transaction](transactions.html#transaction-retries) in case of read/write contention. CockroachDB provides a generic retry function that runs inside a transaction and retries it as needed. You can copy and paste the retry function from here into your code. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ rust +{% include {{ page.version.version }}/app/insecure/txn-sample.rs %} +~~~ + +After running the code, use the [built-in SQL client](cockroach-sql.html) to verify that funds were transferred from one account to another: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e 'SELECT id, balance FROM accounts' --database=bank +~~~ + +~~~ ++----+---------+ +| id | balance | ++----+---------+ +| 1 | 900 | +| 2 | 350 | ++----+---------+ +(2 rows) +~~~ + +
+ +## What's next? + +Read more about using the [Rust Postgres driver](https://crates.io/crates/postgres/). + +{% include {{ page.version.version }}/app/see-also-links.md %} diff --git a/v20.2/build-an-app-with-cockroachdb.md b/v20.2/build-an-app-with-cockroachdb.md new file mode 100644 index 00000000000..7549d7e5561 --- /dev/null +++ b/v20.2/build-an-app-with-cockroachdb.md @@ -0,0 +1,21 @@ +--- +title: Build an App with CockroachDB +summary: The tutorials in this section show you how to build a simple application with CockroachDB, using PostgreSQL-compatible client drivers and ORMs. +tags: golang, python, java +toc: true +twitter: false +--- + +The tutorials in this section show you how to build a simple application with CockroachDB using PostgreSQL-compatible client drivers and ORMs. + +## Tutorials + +{% include {{page.version.version}}/misc/drivers.md %} + +## See also + +- [Client drivers](install-client-drivers.html) +- [Third party database tools](third-party-database-tools.html) +- [Connection parameters](connection-parameters.html) +- [Transactions](transactions.html) +- [Performance best practices](performance-best-practices-overview.html) diff --git a/v20.2/bytes.md b/v20.2/bytes.md new file mode 100644 index 00000000000..88de70a795e --- /dev/null +++ b/v20.2/bytes.md @@ -0,0 +1,126 @@ +--- +title: BYTES +summary: The BYTES data type stores binary strings of variable length. +toc: true +--- + +The `BYTES` [data type](data-types.html) stores binary strings of variable length. + + +## Aliases + +In CockroachDB, the following are aliases for `BYTES`: + +- `BYTEA` +- `BLOB` + +## Syntax + +To express a byte array constant, see the section on +[byte array literals](sql-constants.html#byte-array-literals) for more +details. For example, the following three are equivalent literals for the same +byte array: `b'abc'`, `b'\141\142\143'`, `b'\x61\x62\x63'`. + +In addition to this syntax, CockroachDB also supports using +[string literals](sql-constants.html#string-literals), including the +syntax `'...'`, `e'...'` and `x'....'` in contexts where a byte array +is otherwise expected. + +## Size + +The size of a `BYTES` value is variable, but it's recommended to keep values under 1 MB to ensure performance. Above that threshold, [write amplification](https://en.wikipedia.org/wiki/Write_amplification) and other considerations may cause significant performance degradation. + +## Example + +~~~ sql +> CREATE TABLE bytes (a INT PRIMARY KEY, b BYTES); + +> -- explicitly typed BYTES literals +> INSERT INTO bytes VALUES (1, b'\141\142\143'), (2, b'\x61\x62\x63'), (3, b'\141\x62\c'); + +> -- string literal implicitly typed as BYTES +> INSERT INTO bytes VALUES (4, 'abc'); + + +> SELECT * FROM bytes; +~~~ +~~~ ++---+-----+ +| a | b | ++---+-----+ +| 1 | abc | +| 2 | abc | +| 3 | abc | +| 4 | abc | ++---+-----+ +(4 rows) +~~~ + +## Supported conversions + +`BYTES` values can be +[cast](data-types.html#data-type-conversions-and-casts) explicitly to +[`STRING`](string.html). This conversion always succeeds. Two +conversion modes are supported, controlled by the +[session variable](set-vars.html#supported-variables) `bytea_output`: + +- `hex` (default): The output of the conversion starts with the two + characters `\`, `x` and the rest of the string is composed by the + hexadecimal encoding of each byte in the input. For example, + `x'48AA'::STRING` produces `'\x48AA'`. + +- `escape`: The output of the conversion contains each byte in the + input, as-is if it is an ASCII character, or encoded using the octal + escape format `\NNN` otherwise. For example, `x'48AA'::STRING` + produces `'0\252'`. + +`STRING` values can be cast explicitly to `BYTES`. This conversion +will fail if the hexadecimal digits are not valid, or if there is an +odd number of them. Two conversion modes are supported: + +- If the string starts with the two special characters `\` and `x` + (e.g., `\xAABB`), the rest of the string is interpreted as a sequence + of hexadecimal digits. The string is then converted to a byte array + where each pair of hexadecimal digits is converted to one byte. + +- Otherwise, the string is converted to a byte array that contains its + UTF-8 encoding. + +### `STRING` vs. `BYTES` + +While both `STRING` and `BYTES` can appear to have similar behavior in many situations, one should understand their nuance before casting one into the other. + +`STRING` treats all of its data as characters, or more specifically, Unicode code points. `BYTES` treats all of its data as a byte string. This difference in implementation can lead to dramatically different behavior. For example, let's take a complex Unicode character such as ☃ ([the snowman emoji](https://emojipedia.org/snowman/)): + +{% include copy-clipboard.html %} +~~~ sql +> SELECT length('☃'::string); +~~~ + +~~~ + length ++--------+ + 1 +~~~ + +~~~ sql +> SELECT length('☃'::bytes); +~~~ +~~~ + length ++--------+ + 3 +~~~ + +In this case, [`LENGTH(string)`](functions-and-operators.html#string-and-byte-functions) measures the number of Unicode code points present in the string, whereas [`LENGTH(bytes)`](functions-and-operators.html#string-and-byte-functions) measures the number of bytes required to store that value. Each character (or Unicode code point) can be encoded using multiple bytes, hence the difference in output between the two. + +#### Translating literals to `STRING` vs. `BYTES` + +A literal entered through a SQL client will be translated into a different value based on the type: + ++ `BYTES` give a special meaning to the pair `\x` at the beginning, and translates the rest by substituting pairs of hexadecimal digits to a single byte. For example, `\xff` is equivalent to a single byte with the value of 255. For more information, see [SQL Constants: String literals with character escapes](sql-constants.html#string-literals-with-character-escapes). ++ `STRING` does not give a special meaning to `\x`, so all characters are treated as distinct Unicode code points. For example, `\xff` is treated as a `STRING` with length 4 (`\`, `x`, `f`, and `f`). + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/cancel-job.md b/v20.2/cancel-job.md new file mode 100644 index 00000000000..425530b7fac --- /dev/null +++ b/v20.2/cancel-job.md @@ -0,0 +1,75 @@ +--- +title: CANCEL JOB +summary: The CANCEL JOB statement stops long-running jobs. +toc: true +--- + +The `CANCEL JOB` [statement](sql-statements.html) lets you stop long-running jobs, which include [`IMPORT`](import.html) jobs, enterprise [`BACKUP`](backup.html) and [`RESTORE`](restore.html) jobs, schema changes, [user-created table statistics](create-statistics.html) jobs, [automatic table statistics](cost-based-optimizer.html#table-statistics) jobs, [changefeeds](change-data-capture.html), and [schema change](online-schema-changes.html) jobs. + +## Limitations + +When an enterprise [`RESTORE`](restore.html) is canceled, partially restored data is properly cleaned up. This can have a minor, temporary impact on cluster performance. + +## Required privileges + +Only members of the `admin` role can cancel a job. By default, the `root` user belongs to the `admin` role. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/cancel_job.html %} +
+ +## Parameters + +Parameter | Description +----------|------------ +`job_id` | The ID of the job you want to cancel, which can be found with [`SHOW JOBS`](show-jobs.html). +`select_stmt` | A [selection query](selection-queries.html) that returns `job_id`(s) to cancel. + +## Examples + +### Cancel a single job + +~~~ sql +> SHOW JOBS; +~~~ +~~~ ++----------------+---------+-------------------------------------------+... +| id | type | description |... ++----------------+---------+-------------------------------------------+... +| 27536791415282 | RESTORE | RESTORE db.* FROM 'azure://backup/db/tbl' |... ++----------------+---------+-------------------------------------------+... +~~~ +~~~ sql +> CANCEL JOB 27536791415282; +~~~ + +### Cancel multiple jobs + +To cancel multiple jobs, nest a [`SELECT` clause](select-clause.html) that retrieves `job_id`(s) inside the `CANCEL JOBS` statement: + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL JOBS (SELECT job_id FROM [SHOW JOBS] + WHERE user_name = 'maxroach'); +~~~ + +All jobs created by `maxroach` will be cancelled. + +### Cancel automatic table statistics jobs + +Canceling an automatic table statistics job is not useful since the system will automatically restart the job immediately. To permanently disable automatic table statistics jobs, disable the `sql.stats.automatic_collection.enabled` [cluster setting](cluster-settings.html): + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; +~~~ + +## See also + +- [`SHOW JOBS`](show-jobs.html) +- [`BACKUP`](backup.html) +- [`RESTORE`](restore.html) +- [`IMPORT`](import.html) +- [`CREATE CHANGEFEED`](create-changefeed.html) diff --git a/v20.2/cancel-query.md b/v20.2/cancel-query.md new file mode 100644 index 00000000000..c8f4a84d3c3 --- /dev/null +++ b/v20.2/cancel-query.md @@ -0,0 +1,81 @@ +--- +title: CANCEL QUERY +summary: The CANCEL QUERY statement cancels a running SQL query. +toc: true +--- + +The `CANCEL QUERY` [statement](sql-statements.html) cancels a running SQL query. + + +## Considerations + +- Schema changes are treated differently than other SQL queries. You can use SHOW JOBS to monitor the progress of schema changes and CANCEL JOB to cancel schema changes that are taking longer than expected. +- In rare cases where a query is close to completion when a cancellation request is issued, the query may run to completion. + +## Required privileges + +Members of the `admin` role (include `root`, which belongs to `admin` by default) can cancel any currently active. User that are not members of the `admin` role can cancel only their own currently active queries. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/cancel_query.html %} +
+ +## Parameters + +Parameter | Description +----------|------------ +`query_id` | A [scalar expression](scalar-expressions.html) that produces the ID of the query to cancel.

`CANCEL QUERY` accepts a single query ID. If a subquery is used and returns multiple IDs, the `CANCEL QUERY` statement will fail. To cancel multiple queries, use `CANCEL QUERIES`. +`select_stmt` | A [selection query](selection-queries.html) whose result you want to cancel. + +## Response + +When a query is successfully cancelled, CockroachDB sends a `query execution canceled` error to the client that issued the query. + +- If the canceled query was a single, stand-alone statement, no further action is required by the client. +- If the canceled query was part of a larger, multi-statement [transaction](transactions.html), the client should then issue a [`ROLLBACK`](rollback-transaction.html) statement. + +## Examples + +### Cancel a query via the query ID + +In this example, we use the [`SHOW QUERIES`](show-queries.html) statement to get the ID of a query and then pass the ID into the `CANCEL QUERY` statement: + +~~~ sql +> SHOW QUERIES; +~~~ + +~~~ ++----------------------------------+---------+----------+----------------------------------+----------------------------------+--------------------+------------------+-------------+-----------+ +| query_id | node_id | username | start | query | client_address | application_name | distributed | phase | ++----------------------------------+---------+----------+----------------------------------+----------------------------------+--------------------+------------------+-------------+-----------+ +| 14dacc1f9a781e3d0000000000000001 | 2 | mroach | 2017-08-10 14:08:22.878113+00:00 | SELECT * FROM test.kv ORDER BY k | 192.168.0.72:56194 | test_app | false | executing | ++----------------------------------+---------+----------+----------------------------------+----------------------------------+--------------------+------------------+-------------+-----------+ +| 14dacc206c47a9690000000000000002 | 2 | root | 2017-08-14 19:11:05.309119+00:00 | SHOW CLUSTER QUERIES | 127.0.0.1:50921 | | NULL | preparing | ++----------------------------------+---------+----------+----------------------------------+----------------------------------+--------------------+------------------+-------------+-----------+ +~~~ + +~~~ sql +> CANCEL QUERY '14dacc1f9a781e3d0000000000000001'; +~~~ + +### Cancel a query via a subquery + +In this example, we nest a [`SELECT` clause](select-clause.html) that retrieves the ID of a query inside the `CANCEL QUERY` statement: + +~~~ sql +> CANCEL QUERY (SELECT query_id FROM [SHOW CLUSTER QUERIES] + WHERE client_address = '192.168.0.72:56194' + AND username = 'mroach' + AND query = 'SELECT * FROM test.kv ORDER BY k'); +~~~ + +{{site.data.alerts.callout_info}}CANCEL QUERY accepts a single query ID. If a subquery is used and returns multiple IDs, the CANCEL QUERY statement will fail. To cancel multiple queries, use CANCEL QUERIES.{{site.data.alerts.end}} + +## See also + +- [Manage Long-Running Queries](manage-long-running-queries.html) +- [`SHOW QUERIES`](show-queries.html) +- [`CANCEL SESSION`](cancel-session.html) +- [SQL Statements](sql-statements.html) diff --git a/v20.2/cancel-session.md b/v20.2/cancel-session.md new file mode 100644 index 00000000000..c50142f1ca9 --- /dev/null +++ b/v20.2/cancel-session.md @@ -0,0 +1,93 @@ +--- +title: CANCEL SESSION +summary: The CANCEL SESSION statement stops long-running sessions. +toc: true +--- + +The `CANCEL SESSION` [statement](sql-statements.html) lets you stop long-running sessions. `CANCEL SESSION` will attempt to cancel the currently active query and end the session. + + +## Required privileges + +Only members of the `admin` role and the user that the session belongs to can cancel a session. By default, the `root` user belongs to the `admin` role. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/cancel_session.html %}
+ +## Parameters + +Parameter | Description +----------|------------ +`session_id` | The ID of the session you want to cancel, which can be found with [`SHOW SESSIONS`](show-sessions.html).

`CANCEL SESSION` accepts a single session ID. If a subquery is used and returns multiple IDs, the `CANCEL SESSION` statement will fail. To cancel multiple sessions, use `CANCEL SESSIONS`. +`select_stmt` | A [selection query](selection-queries.html) that returns `session_id`(s) to cancel. + +## Example + +### Cancel a single session + +In this example, we use the [`SHOW SESSIONS`](show-sessions.html) statement to get the ID of a session and then pass the ID into the `CANCEL SESSION` statement: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW SESSIONS; +~~~ +~~~ ++---------+----------------------------------+-----------+... +| node_id | session_id | user_name |... ++---------+----------------------------------+-----------+... +| 1 | 1530c309b1d8d5f00000000000000001 | root |... ++---------+----------------------------------+-----------+... +| 1 | 1530fe0e46d2692e0000000000000001 | maxroach |... ++---------+----------------------------------+-----------+... +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL SESSION '1530fe0e46d2692e0000000000000001'; +~~~ + +You can also cancel a session using a subquery that returns a single session ID: + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL SESSIONS (SELECT session_id FROM [SHOW SESSIONS] + WHERE username = 'root'); +~~~ + +### Cancel multiple sessions + +Use the [`SHOW SESSIONS`](show-sessions.html) statement to view all active sessions: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW SESSIONS; +~~~ +~~~ ++---------+----------------------------------+-----------+... +| node_id | session_id | user_name |... ++---------+----------------------------------+-----------+... +| 1 | 1530c309b1d8d5f00000000000000001 | root |... ++---------+----------------------------------+-----------+... +| 1 | 1530fe0e46d2692e0000000000000001 | maxroach |... ++---------+----------------------------------+-----------+... +| 1 | 15310cc79671fc6a0000000000000001 | maxroach |... ++---------+----------------------------------+-----------+... +~~~ + +To cancel multiple sessions, nest a [`SELECT` clause](select-clause.html) that retrieves `session_id`(s) inside the `CANCEL SESSIONS` statement: + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL SESSIONS (SELECT session_id FROM [SHOW SESSIONS] + WHERE user_name = 'maxroach'); +~~~ + +All sessions created by `maxroach` will be cancelled. + +## See also + +- [`SHOW SESSIONS`](show-sessions.html) +- [`SET` (session variable)](set-vars.html) +- [`SHOW` (session variable)](show-vars.html) +- [SQL Statements](sql-statements.html) diff --git a/v20.2/change-data-capture.md b/v20.2/change-data-capture.md new file mode 100644 index 00000000000..3c4eca1c9bf --- /dev/null +++ b/v20.2/change-data-capture.md @@ -0,0 +1,805 @@ +--- +title: Change Data Capture (CDC) +summary: Change data capture (CDC) provides efficient, distributed, row-level change subscriptions. +toc: true +--- + +Change data capture (CDC) provides efficient, distributed, row-level change feeds into a configurable sink for downstream processing such as reporting, caching, or full-text indexing. + +## What is change data capture? + +While CockroachDB is an excellent system of record, it also needs to coexist with other systems. For example, you might want to keep your data mirrored in full-text indexes, analytics engines, or big data pipelines. + +The main feature of CDC is the changefeed, which targets an allowlist of tables, called the "watched rows". There are two implementations of changefeeds: + +- [Core changefeeds](#create-a-core-changefeed), which stream row-level changes to the client indefinitely until the underlying connection is closed or the changefeed is canceled. +- [Enterprise changefeeds](#configure-a-changefeed-enterprise), where every change to a watched row is emitted as a record in a configurable format (`JSON` or Avro) to a configurable sink ([Kafka](https://kafka.apache.org/)). + +## Ordering guarantees + +- In most cases, each version of a row will be emitted once. However, some infrequent conditions (e.g., node failures, network partitions) will cause them to be repeated. This gives our changefeeds an **at-least-once delivery guarantee**. + +- Once a row has been emitted with some timestamp, no previously unseen versions of that row will be emitted with a lower timestamp. That is, you will never see a _new_ change for that row at an earlier timestamp. + + For example, if you ran the following: + + ~~~ sql + > CREATE TABLE foo (id INT PRIMARY KEY DEFAULT unique_rowid(), name STRING); + > CREATE CHANGEFEED FOR TABLE foo INTO 'kafka://localhost:9092' WITH UPDATED; + > INSERT INTO foo VALUES (1, 'Carl'); + > UPDATE foo SET name = 'Petee' WHERE id = 1; + ~~~ + + You'd expect the changefeed to emit: + + ~~~ shell + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Carl"} + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Petee"} + ~~~ + + It is also possible that the changefeed emits an out of order duplicate of an earlier value that you already saw: + + ~~~ shell + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Carl"} + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Petee"} + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Carl"} + ~~~ + + However, you will **never** see an output like the following (i.e., an out of order row that you've never seen before): + + ~~~ shell + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Petee"} + [1] {"__crdb__": {"updated": }, "id": 1, "name": "Carl"} + ~~~ + +- If a row is modified more than once in the same transaction, only the last change will be emitted. + +- Rows are sharded between Kafka partitions by the row’s [primary key](primary-key.html). + +- The `UPDATED` option adds an "updated" timestamp to each emitted row. You can also use the `RESOLVED` option to emit periodic "resolved" timestamp messages to each Kafka partition. A "resolved" timestamp is a guarantee that no (previously unseen) rows with a lower update timestamp will be emitted on that partition. + + For example: + + ~~~ shell + {"__crdb__": {"updated": "1532377312562986715.0000000000"}, "id": 1, "name": "Petee H"} + {"__crdb__": {"updated": "1532377306108205142.0000000000"}, "id": 2, "name": "Carl"} + {"__crdb__": {"updated": "1532377358501715562.0000000000"}, "id": 3, "name": "Ernie"} + {"__crdb__":{"resolved":"1532379887442299001.0000000000"}} + {"__crdb__":{"resolved":"1532379888444290910.0000000000"}} + {"__crdb__":{"resolved":"1532379889448662988.0000000000"}} + ... + {"__crdb__":{"resolved":"1532379922512859361.0000000000"}} + {"__crdb__": {"updated": "1532379923319195777.0000000000"}, "id": 4, "name": "Lucky"} + ~~~ + +- With duplicates removed, an individual row is emitted in the same order as the transactions that updated it. However, this is not true for updates to two different rows, even two rows in the same table. + + To compare two different rows for [happens-before](https://en.wikipedia.org/wiki/Happened-before), compare the "updated" timestamp. This works across anything in the same cluster (e.g., tables, nodes, etc.). + + Resolved timestamp notifications on every Kafka partition can be used to provide strong ordering and global consistency guarantees by buffering records in between timestamp closures. Use the "resolved" timestamp to see every row that changed at a certain time. + + The complexity with timestamps is necessary because CockroachDB supports transactions that can affect any part of the cluster, and it is not possible to horizontally divide the transaction log into independent changefeeds. For more information about this, [read our blog post on CDC](https://www.cockroachlabs.com/blog/change-data-capture/). + +## Avro schema changes + +To ensure that the Avro schemas that CockroachDB publishes will work with the schema compatibility rules used by the Confluent schema registry, CockroachDB emits all fields in Avro as nullable unions. This ensures that Avro and Confluent consider the schemas to be both backward- and forward-compatible, since the Confluent Schema Registry has a different set of rules than Avro for schemas to be backward- and forward-compatible. + +Note that the original CockroachDB column definition is also included in the schema as a doc field, so it's still possible to distinguish between a `NOT NULL` CockroachDB column and a `NULL` CockroachDB column. + +## Schema changes with column backfill + +When schema changes with column backfill (e.g., adding a column with a default, adding a computed column, adding a `NOT NULL` column, dropping a column) are made to watched rows, the changefeed will emit some duplicates during the backfill. When it finishes, CockroachDB outputs all watched rows using the new schema. When using Avro, rows that have been backfilled by a schema change are always re-emitted. + +For an example of a schema change with column backfill, start with the changefeed created in the [example below](#create-a-changefeed-connected-to-kafka): + +~~~ shell +[1] {"id": 1, "name": "Petee H"} +[2] {"id": 2, "name": "Carl"} +[3] {"id": 3, "name": "Ernie"} +~~~ + +Add a column to the watched table: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE office_dogs ADD COLUMN likes_treats BOOL DEFAULT TRUE; +~~~ + +The changefeed emits duplicate records 1, 2, and 3 before outputting the records using the new schema: + +~~~ shell +[1] {"id": 1, "name": "Petee H"} +[2] {"id": 2, "name": "Carl"} +[3] {"id": 3, "name": "Ernie"} +[1] {"id": 1, "name": "Petee H"} # Duplicate +[2] {"id": 2, "name": "Carl"} # Duplicate +[3] {"id": 3, "name": "Ernie"} # Duplicate +[1] {"id": 1, "likes_treats": true, "name": "Petee H"} +[2] {"id": 2, "likes_treats": true, "name": "Carl"} +[3] {"id": 3, "likes_treats": true, "name": "Ernie"} +~~~ + +## Enable rangefeeds to reduce latency + +Previously created changefeeds collect changes by periodically sending a request for any recent changes. Newly created changefeeds now behave differently: they connect to a long-lived request (i.e., a rangefeed), which pushes changes as they happen. This reduces the latency of row changes, as well as reduces transaction restarts on tables being watched by a changefeed for some workloads. + +To enable rangefeeds, set the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html) to `true`. Any created changefeed will error until this setting is enabled. Note that enabling rangefeeds currently has a small performance cost (about a 5-10% increase in latencies), whether or not the rangefeed is being using in a changefeed. + +If you are experiencing an issue, you can revert back to the previous behavior by setting `changefeed.push.enabled` to `false`. Note that this setting will be removed in a future release; if you have to use the fallback, please [file a Github issue](file-an-issue.html). + +{{site.data.alerts.callout_info}} +To enable rangefeeds for an existing changefeed, you must also restart the changefeed. For an enterprise changefeed, [pause](#pause) and [resume](#resume) the changefeed. For a core changefeed, cut the connection (**CTRL+C**) and reconnect using the `cursor` option. +{{site.data.alerts.end}} + +The `kv.closed_timestamp.target_duration` [cluster setting](cluster-settings.html) can be used with push changefeeds. Resolved timestamps will always be behind by at least this setting's duration; however, decreasing the duration leads to more transaction restarts in your cluster, which can affect performance. + +## Create a changefeed (Core) + +A core changefeed streams row-level changes to the client indefinitely until the underlying connection is closed or the changefeed is canceled. + +To create a core changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> EXPERIMENTAL CHANGEFEED FOR name; +~~~ + +For more information, see [`CHANGEFEED FOR`](changefeed-for.html). + +## Configure a changefeed (Enterprise) + +An enterprise changefeed streams row-level changes in a configurable format to a configurable sink (i.e., Kafka or a cloud storage sink). You can [create](#create), [pause](#pause), [resume](#resume), [cancel](#cancel), [monitor](#monitor-a-changefeed), and [debug](#debug-a-changefeed) an enterprise changefeed. + +### Create + +To create an enterprise changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE CHANGEFEED FOR TABLE table_name, table_name2 INTO 'scheme://host:port'; +~~~ + +For more information, see [`CREATE CHANGEFEED`](create-changefeed.html). + +### Pause + +To pause an enterprise changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> PAUSE JOB job_id; +~~~ + +For more information, see [`PAUSE JOB`](pause-job.html). + +### Resume + +To resume a paused enterprise changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> RESUME JOB job_id; +~~~ + +For more information, see [`RESUME JOB`](resume-job.html). + +### Cancel + +To cancel an enterprise changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL JOB job_id; +~~~ + +For more information, see [`CANCEL JOB`](cancel-job.html). + +## Monitor a changefeed + +{{site.data.alerts.callout_info}} +Monitoring is only available for enterprise changefeeds. +{{site.data.alerts.end}} + +Changefeed progress is exposed as a high-water timestamp that advances as the changefeed progresses. This is a guarantee that all changes before or at the timestamp have been emitted. You can monitor a changefeed: + +- On the [Changefeed Dashboard](admin-ui-cdc-dashboard.html) of the Admin UI. +- On the [Jobs page](admin-ui-jobs-page.html) of the Admin UI. Hover over the high-water timestamp to view the [system time](as-of-system-time.html). +- Using `crdb_internal.jobs`: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM crdb_internal.jobs WHERE job_id = ; + ~~~ + ~~~ + job_id | job_type | description | ... | high_water_timestamp | error | coordinator_id + +--------------------+------------+------------------------------------------------------------------------+ ... +--------------------------------+-------+----------------+ + 383870400694353921 | CHANGEFEED | CREATE CHANGEFEED FOR TABLE office_dogs INTO 'kafka://localhost:9092' | ... | 1537279405671006870.0000000000 | | 1 + (1 row) + ~~~ + +- Setting up an alert on the `changefeed.max_behind_nanos` metric to track when a changefeed's high-water mark timestamp is at risk of falling behind the cluster's [garbage collection window](configure-replication-zones.html#replication-zone-variables). For more information, see [Monitoring and Alerting](monitoring-and-alerting.html#changefeed-is-experiencing-high-latency). + +{{site.data.alerts.callout_info}} +You can use the high-water timestamp to [start a new changefeed where another ended](create-changefeed.html#start-a-new-changefeed-where-another-ended). +{{site.data.alerts.end}} + +## Debug a changefeed + +### Using logs + +For enterprise changefeeds, [use log information](debug-and-error-logs.html) to debug connection issues (i.e., `kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`). Debug by looking for lines in the logs with `[kafka-producer]` in them: + +~~~ +I190312 18:56:53.535646 585 vendor/github.com/Shopify/sarama/client.go:123 [kafka-producer] Initializing new client +I190312 18:56:53.535714 585 vendor/github.com/Shopify/sarama/client.go:724 [kafka-producer] client/metadata fetching metadata for all topics from broker localhost:9092 +I190312 18:56:53.536730 569 vendor/github.com/Shopify/sarama/broker.go:148 [kafka-producer] Connected to broker at localhost:9092 (unregistered) +I190312 18:56:53.537661 585 vendor/github.com/Shopify/sarama/client.go:500 [kafka-producer] client/brokers registered new broker #0 at 172.16.94.87:9092 +I190312 18:56:53.537686 585 vendor/github.com/Shopify/sarama/client.go:170 [kafka-producer] Successfully initialized new client +~~~ + +### Using `SHOW JOBS` + +For enterprise changefeeds, you can check the status by using: + +{% include copy-clipboard.html %} +~~~ sql +SELECT * FROM [SHOW JOBS] WHERE job_type='CHANGEFEED'; +~~~ + +Or: + +{% include copy-clipboard.html %} +~~~ sql +SELECT * from crdb_internal.jobs WHERE job_type='CHANGEFEED'; +~~~ + +For more information, see [`SHOW JOBS`](show-jobs.html). + +### Using the Admin UI + + On the [**Custom Chart** debug page](admin-ui-custom-chart-debug-page.html) of the Admin UI: + +1. To add a chart, click **Add Chart**. +2. Select `changefeed.error_retries` from the **Metric Name** dropdown menu. + + A graph of changefeed restarts due to retryable errors will display. + +## Usage examples + +### Create a core changefeed + +{% include {{ page.version.version }}/cdc/create-core-changefeed.md %} + +### Create a core changefeed using Avro + +{% include {{ page.version.version }}/cdc/create-core-changefeed-avro.md %} + +### Create a changefeed connected to Kafka + +{{site.data.alerts.callout_info}} +[`CREATE CHANGEFEED`](create-changefeed.html) is an [enterprise-only](enterprise-licensing.html) feature. For the core version, see [the `CHANGEFEED FOR` example above](#create-a-core-changefeed). +{{site.data.alerts.end}} + +In this example, you'll set up a changefeed for a single-node cluster that is connected to a Kafka sink. The changefeed will watch two tables. + +1. If you do not already have one, [request a trial enterprise license](enterprise-licensing.html). + +2. Use the [`cockroach start-single-node`](cockroach-start-single-node.html) command to start a single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --insecure --listen-addr=localhost --background + ~~~ + +3. Download and extract the [Confluent Open Source platform](https://www.confluent.io/download/) (which includes Kafka). + +4. Move into the extracted `confluent-` directory and start Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent start + ~~~ + + Only `zookeeper` and `kafka` are needed. To troubleshoot Confluent, see [their docs](https://docs.confluent.io/current/installation/installing_cp.html#zip-and-tar-archives). + +5. Create two Kafka topics: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-topics \ + --create \ + --zookeeper localhost:2181 \ + --replication-factor 1 \ + --partitions 1 \ + --topic office_dogs + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-topics \ + --create \ + --zookeeper localhost:2181 \ + --replication-factor 1 \ + --partitions 1 \ + --topic employees + ~~~ + + {{site.data.alerts.callout_info}} + You are expected to create any Kafka topics with the necessary number of replications and partitions. [Topics can be created manually](https://kafka.apache.org/documentation/#basic_ops_add_topic) or [Kafka brokers can be configured to automatically create topics](https://kafka.apache.org/documentation/#topicconfigs) with a default partition count and replication factor. + {{site.data.alerts.end}} + +6. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +7. Set your organization name and [enterprise license](enterprise-licensing.html) key that you received via email: + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING cluster.organization = ''; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING enterprise.license = ''; + ~~~ + +8. Enable the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING kv.rangefeed.enabled = true; + ~~~ + +9. Create a database called `cdc_demo`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE cdc_demo; + ~~~ + +10. Set the database as the default: + + {% include copy-clipboard.html %} + ~~~ sql + > SET DATABASE = cdc_demo; + ~~~ + +11. Create a table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE office_dogs ( + id INT PRIMARY KEY, + name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO office_dogs VALUES + (1, 'Petee'), + (2, 'Carl'); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE office_dogs SET name = 'Petee H' WHERE id = 1; + ~~~ + +12. Create another table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE employees ( + dog_id INT REFERENCES office_dogs (id), + employee_name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO employees VALUES + (1, 'Lauren'), + (2, 'Spencer'); + ~~~ + +13. Start the changefeed: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE CHANGEFEED FOR TABLE office_dogs, employees INTO 'kafka://localhost:9092'; + ~~~ + ~~~ + + job_id + +--------------------+ + 360645287206223873 + (1 row) + ~~~ + + This will start up the changefeed in the background and return the `job_id`. The changefeed writes to Kafka. + +14. In a new terminal, move into the extracted `confluent-` directory and start watching the Kafka topics: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-console-consumer \ + --bootstrap-server=localhost:9092 \ + --from-beginning \ + --whitelist 'office_dogs|employees' + ~~~ + + ~~~ shell + {"after": {"id": 1, "name": "Petee H"}} + {"after": {"id": 2, "name": "Carl"}} + {"after": {"id": 1, "name": "Lauren", "rowid": 528514320239329281}} + {"after": {"id": 2, "name": "Spencer", "rowid": 528514320239362049}} + ~~~ + + The initial scan displays the state of the tables as of when the changefeed started (therefore, the initial value of `"Petee"` is omitted). + + {% include {{ page.version.version }}/cdc/print-key.md %} + +15. Back in the SQL client, insert more data: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO office_dogs VALUES (3, 'Ernie'); + ~~~ + +16. Back in the terminal where you're watching the Kafka topics, the following output has appeared: + + ~~~ shell + {"after": {"id": 3, "name": "Ernie"}} + ~~~ + +17. When you are done, exit the SQL shell (`\q`). + +18. To stop `cockroach`, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure + ~~~ + +19. To stop Kafka, move into the extracted `confluent-` directory and stop Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent stop + ~~~ + +### Create a changefeed connected to Kafka using Avro + +{{site.data.alerts.callout_info}} +[`CREATE CHANGEFEED`](create-changefeed.html) is an [enterprise-only](enterprise-licensing.html) feature. For the core version, see [the `CHANGEFEED FOR` example above](#create-a-core-changefeed-using-avro). +{{site.data.alerts.end}} + +In this example, you'll set up a changefeed for a single-node cluster that is connected to a Kafka sink and emits [Avro](https://avro.apache.org/docs/1.8.2/spec.html) records. The changefeed will watch two tables. + +1. If you do not already have one, [request a trial enterprise license](enterprise-licensing.html). + +2. Use the [`cockroach start-single-node`](cockroach-start-single-node.html) command to start a single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --insecure --listen-addr=localhost --background + ~~~ + +3. Download and extract the [Confluent Open Source platform](https://www.confluent.io/download/) (which includes Kafka). + +4. Move into the extracted `confluent-` directory and start Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent start + ~~~ + + Only `zookeeper`, `kafka`, and `schema-registry` are needed. To troubleshoot Confluent, see [their docs](https://docs.confluent.io/current/installation/installing_cp.html#zip-and-tar-archives). + +5. Create two Kafka topics: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-topics \ + --create \ + --zookeeper localhost:2181 \ + --replication-factor 1 \ + --partitions 1 \ + --topic office_dogs + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-topics \ + --create \ + --zookeeper localhost:2181 \ + --replication-factor 1 \ + --partitions 1 \ + --topic employees + ~~~ + + {{site.data.alerts.callout_info}} + You are expected to create any Kafka topics with the necessary number of replications and partitions. [Topics can be created manually](https://kafka.apache.org/documentation/#basic_ops_add_topic) or [Kafka brokers can be configured to automatically create topics](https://kafka.apache.org/documentation/#topicconfigs) with a default partition count and replication factor. + {{site.data.alerts.end}} + +6. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +7. Set your organization name and [enterprise license](enterprise-licensing.html) key that you received via email: + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING cluster.organization = ''; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING enterprise.license = ''; + ~~~ + +8. Enable the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING kv.rangefeed.enabled = true; + ~~~ + +9. Create a database called `cdc_demo`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE cdc_demo; + ~~~ + +10. Set the database as the default: + + {% include copy-clipboard.html %} + ~~~ sql + > SET DATABASE = cdc_demo; + ~~~ + +11. Create a table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE office_dogs ( + id INT PRIMARY KEY, + name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO office_dogs VALUES + (1, 'Petee'), + (2, 'Carl'); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE office_dogs SET name = 'Petee H' WHERE id = 1; + ~~~ + +12. Create another table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE employees ( + dog_id INT REFERENCES office_dogs_avro (id), + employee_name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO employees VALUES + (1, 'Lauren'), + (2, 'Spencer'); + ~~~ + +13. Start the changefeed: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE CHANGEFEED FOR TABLE office_dogs, employees INTO 'kafka://localhost:9092' WITH format = experimental_avro, confluent_schema_registry = 'http://localhost:8081'; + ~~~ + + ~~~ + job_id + +--------------------+ + 360645287206223873 + (1 row) + ~~~ + + This will start up the changefeed in the background and return the `job_id`. The changefeed writes to Kafka. + +14. In a new terminal, move into the extracted `confluent-` directory and start watching the Kafka topics: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/kafka-avro-console-consumer \ + --bootstrap-server=localhost:9092 \ + --from-beginning \ + --whitelist 'office_dogs|employees' + ~~~ + + ~~~ shell + {"after":{"office_dogs":{"id":{"long":1},"name":{"string":"Petee H"}}}} + {"after":{"office_dogs":{"id":{"long":2},"name":{"string":"Carl"}}}} + {"after":{"employees":{"dog_id":{"long":1},"employee_name":{"string":"Lauren"},"rowid":{"long":528537452042682369}}}} + {"after":{"employees":{"dog_id":{"long":2},"employee_name":{"string":"Spencer"},"rowid":{"long":528537452042747905}}}} + ~~~ + + The initial scan displays the state of the table as of when the changefeed started (therefore, the initial value of `"Petee"` is omitted). + + {% include {{ page.version.version }}/cdc/print-key.md %} + +15. Back in the SQL client, insert more data: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO office_dogs VALUES (3, 'Ernie'); + ~~~ + +16. Back in the terminal where you're watching the Kafka topics, the following output has appeared: + + ~~~ shell + {"after":{"office_dogs":{"id":{"long":3},"name":{"string":"Ernie"}}}} + ~~~ + +17. When you are done, exit the SQL shell (`\q`). + +18. To stop `cockroach`, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure + ~~~ + +19. To stop Kafka, move into the extracted `confluent-` directory and stop Confluent: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./bin/confluent stop + ~~~ + +### Create a changefeed connected to a cloud storage sink + +{{site.data.alerts.callout_info}} +[`CREATE CHANGEFEED`](create-changefeed.html) is an [enterprise-only](enterprise-licensing.html) feature. For the core version, see [the `CHANGEFEED FOR` example above](#create-a-core-changefeed). +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +In this example, you'll set up a changefeed for a single-node cluster that is connected to an AWS S3 sink. The changefeed watches two tables. Note that you can set up changefeeds for any of [these cloud storage providers](create-changefeed.html#cloud-storage-sink). + +1. If you do not already have one, [request a trial enterprise license](enterprise-licensing.html). + +2. Use the [`cockroach start-single-node`](cockroach-start-single-node.html) command to start a single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --insecure --listen-addr=localhost --background + ~~~ + +3. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +4. Set your organization name and [enterprise license](enterprise-licensing.html) key that you received via email: + + {% include copy-clipboard.html %} + ~~~ shell + > SET CLUSTER SETTING cluster.organization = ''; + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + > SET CLUSTER SETTING enterprise.license = ''; + ~~~ + +5. Enable the `kv.rangefeed.enabled` [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING kv.rangefeed.enabled = true; + ~~~ + +6. Create a database called `cdc_demo`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE cdc_demo; + ~~~ + +7. Set the database as the default: + + {% include copy-clipboard.html %} + ~~~ sql + > SET DATABASE = cdc_demo; + ~~~ + +8. Create a table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE office_dogs ( + id INT PRIMARY KEY, + name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO office_dogs VALUES + (1, 'Petee'), + (2, 'Carl'); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE office_dogs SET name = 'Petee H' WHERE id = 1; + ~~~ + +9. Create another table and add data: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE employees ( + dog_id INT REFERENCES office_dogs_avro (id), + employee_name STRING); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO employees VALUES + (1, 'Lauren'), + (2, 'Spencer'); + ~~~ + +10. Start the changefeed: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE CHANGEFEED FOR TABLE office_dogs, employees INTO 'experimental-s3://example-bucket-name/test?AWS_ACCESS_KEY_ID=enter_key-here&AWS_SECRET_ACCESS_KEY=enter_key_here' with updated, resolved='10s'; + ~~~ + + ~~~ + job_id + +--------------------+ + 360645287206223873 + (1 row) + ~~~ + + This will start up the changefeed in the background and return the `job_id`. The changefeed writes to AWS. + +11. Monitor your changefeed on the [Admin UI](http://localhost:8080/#/metrics/changefeeds/cluster). For more information, see [Changefeeds Dashboard](admin-ui-cdc-dashboard.html). + +12. When you are done, exit the SQL shell (`\q`). + +13. To stop `cockroach`, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure + ~~~ + +## Known limitations + +{% include {{ page.version.version }}/known-limitations/cdc.md %} + +## See also + +- [`CREATE CHANGEFEED`](create-changefeed.html) +- [`CHANGEFEED FOR`](changefeed-for.html) +- [`PAUSE JOB`](pause-job.html) +- [`CANCEL JOB`](cancel-job.html) +- [Other SQL Statements](sql-statements.html) +- [Changefeed Dashboard](admin-ui-cdc-dashboard.html) diff --git a/v20.2/changefeed-for.md b/v20.2/changefeed-for.md new file mode 100644 index 00000000000..13279fc0a6f --- /dev/null +++ b/v20.2/changefeed-for.md @@ -0,0 +1,80 @@ +--- +title: EXPERIMENTAL CHANGEFEED FOR +summary: which streams row-level changes to the client indefinitely until the underlying connection is closed or the changefeed is canceled. +toc: true +--- + +{{site.data.alerts.callout_info}} +`EXPERIMENTAL CHANGEFEED FOR` is the core implementation of changefeeds. For the [enterprise-only](enterprise-licensing.html) version, see [`CREATE CHANGEFEED`](create-changefeed.html). +{{site.data.alerts.end}} + +The `EXPERIMENTAL CHANGEFEED FOR` [statement](sql-statements.html) creates a new core changefeed, which streams row-level changes to the client indefinitely until the underlying connection is closed or the changefeed is canceled. + +{% include {{ page.version.version }}/cdc/core-url.md %} + +For more information, see [Change Data Capture](change-data-capture.html). + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +## Required privileges + +Changefeeds can only be created by superusers, i.e., [members of the `admin` role](authorization.html#create-and-manage-roles). The admin role exists by default with `root` as the member. + +## Synopsis + +~~~ +> EXPERIMENTAL CHANGEFEED FOR table_name [ WITH (option [= value] [, ...]) ]; +~~~ + +## Parameters + +Parameter | Description +----------|------------ +`table_name` | The name of the table (or tables in a comma separated list) to create a changefeed for. +`option` / `value` | For a list of available options and their values, see [Options](#options) below. + + + +### Options + +Option | Value | Description +-------|-------|------------ +`updated` | N/A | Include updated timestamps with each row. +`resolved` | [`INTERVAL`](interval.html) | Periodically emit resolved timestamps to the changefeed. Optionally, set a minimum duration between emitting resolved timestamps. If unspecified, all resolved timestamps are emitted.

Example: `resolved='10s'` +`envelope` | `key_only` / `row` | Use `key_only` to emit only the key and no value, which is faster if you only want to know when the key changes.

Default: `envelope=row` +`cursor` | [Timestamp](as-of-system-time.html#parameters) | Emits any changes after the given timestamp, but does not output the current state of the table first. If `cursor` is not specified, the changefeed starts by doing a consistent scan of all the watched rows and emits the current value, then moves to emitting any changes that happen after the scan.

`cursor` can be used to start a new changefeed where a previous changefeed ended.

Example: `CURSOR=1536242855577149065.0000000000` +`format` | `json` / `experimental_avro` | Format of the emitted record. Currently, support for [Avro is limited and experimental](#avro-limitations).

Default: `format=json`. +`confluent_schema_registry` | Schema Registry address | The [Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html#sr) address is required to use `experimental_avro`. + +#### Avro limitations + +Currently, support for Avro is limited and experimental. Below is a list of unsupported SQL types and values for Avro changefeeds: + +- [Decimals](decimal.html) must have precision specified. +- [Decimals](decimal.html) with `NaN` or infinite values cannot be written in Avro. + + {{site.data.alerts.callout_info}} + To avoid `NaN` or infinite values, add a [`CHECK` constraint](check.html) to prevent these values from being inserted into decimal columns. + {{site.data.alerts.end}} + +- [`time`, `date`, `interval`](https://github.com/cockroachdb/cockroach/issues/32472), [`uuid`, `inet`](https://github.com/cockroachdb/cockroach/issues/34417), [`array`](https://github.com/cockroachdb/cockroach/issues/34420), and [`jsonb`](https://github.com/cockroachdb/cockroach/issues/34421) are not supported in Avro yet. + +## Examples + +### Create a changefeed + +{% include {{ page.version.version }}/cdc/create-core-changefeed.md %} + +### Create a changefeed with Avro + +{% include {{ page.version.version }}/cdc/create-core-changefeed-avro.md %} + + + +## See also + +- [Change Data Capture](change-data-capture.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/check.md b/v20.2/check.md new file mode 100644 index 00000000000..976a63394c1 --- /dev/null +++ b/v20.2/check.md @@ -0,0 +1,113 @@ +--- +title: CHECK Constraint +summary: The CHECK constraint specifies that values for the column in INSERT or UPDATE statements must satisfy a Boolean expression. +toc: true +--- + +The `CHECK` [constraint](constraints.html) specifies that values for the column in [`INSERT`](insert.html) or [`UPDATE`](update.html) statements must return `TRUE` or `NULL` for a Boolean expression. If any values return `FALSE`, the entire statement is rejected. + +## Details + +- If you add a `CHECK` constraint to an existing table, CockroachDB will run a background job to validate existing table data in the process of adding the constraint. If a row is found that violates the constraint during the validation step, the [`ADD CONSTRAINT`](add-constraint.html) statement will fail. This differs from previous versions of CockroachDB, which allowed you to add a check constraint that was enforced for writes but could be violated by rows that existed prior to adding the constraint. +- Check constraints can be added to columns that were created earlier in the same transaction. For an example, see [Add the `CHECK` constraint](add-constraint.html#add-the-check-constraint). +- `CHECK` constraints may be specified at the column or table level and can reference other columns within the table. Internally, all column-level `CHECK` constraints are converted to table-level constraints so they can be handled consistently. +- You can have multiple `CHECK` constraints on a single column but ideally, for performance optimization, these should be combined using the logical operators. For example: + + ~~~ sql + warranty_period INT CHECK (warranty_period >= 0) CHECK (warranty_period <= 24) + ~~~ + + should be specified as: + + ~~~ sql + warranty_period INT CHECK (warranty_period BETWEEN 0 AND 24) + ~~~ +- When a column with a `CHECK` constraint is dropped, the `CHECK` constraint is also dropped. + +## Syntax + +`CHECK` constraints can be defined at the [table level](#table-level). However, if you only want the constraint to apply to a single column, it can be applied at the [column level](#column-level). + +{{site.data.alerts.callout_info}}You can also add the CHECK constraint to existing tables through ADD CONSTRAINT.{{site.data.alerts.end}} + +### Column level + +
+ {% include {{ page.version.version }}/sql/diagrams/check_column_level.html %} +
+ + Parameter | Description +-----------|------------- + `table_name` | The name of the table you're creating. + `column_name` | The name of the constrained column. + `column_type` | The constrained column's [data type](data-types.html). + `check_expr` | An expression that returns a Boolean value; if the expression evaluates to `FALSE`, the value cannot be inserted. + `column_constraints` | Any other column-level [constraints](constraints.html) you want to apply to this column. + `column_def` | Definitions for any other columns in the table. + `table_constraints` | Any table-level [constraints](constraints.html) you want to apply. + +**Example** + +~~~ sql +> CREATE TABLE inventories ( + product_id INT NOT NULL, + warehouse_id INT NOT NULL, + quantity_on_hand INT NOT NULL CHECK (quantity_on_hand > 0), + PRIMARY KEY (product_id, warehouse_id) + ); +~~~ + +### Table level + +
+ {% include {{ page.version.version }}/sql/diagrams/check_table_level.html %} +
+ + Parameter | Description +-----------|------------- + `table_name` | The name of the table you're creating. + `column_def` | Definitions for any other columns in the table. + `name` | The name you want to use for the constraint, which must be unique to its table and follow these [identifier rules](keywords-and-identifiers.html#identifiers). + `check_expr` | An expression that returns a Boolean value; if the expression evaluates to `FALSE`, the value cannot be inserted. + `table_constraints` | Any other table-level [constraints](constraints.html) you want to apply. + +**Example** + +~~~ sql +> CREATE TABLE inventories ( + product_id INT NOT NULL, + warehouse_id INT NOT NULL, + quantity_on_hand INT NOT NULL, + PRIMARY KEY (product_id, warehouse_id), + CONSTRAINT ok_to_supply CHECK (quantity_on_hand > 0 AND warehouse_id BETWEEN 100 AND 200) + ); +~~~ + +## Usage example + +`CHECK` constraints may be specified at the column or table level and can reference other columns within the table. Internally, all column-level `CHECK` constraints are converted to table-level constraints so they can be handled in a consistent fashion. + +~~~ sql +> CREATE TABLE inventories ( + product_id INT NOT NULL, + warehouse_id INT NOT NULL, + quantity_on_hand INT NOT NULL CHECK (quantity_on_hand > 0), + PRIMARY KEY (product_id, warehouse_id) + ); + +> INSERT INTO inventories (product_id, warehouse_id, quantity_on_hand) VALUES (1, 2, 0); +~~~ +~~~ +pq: failed to satisfy CHECK constraint (quantity_on_hand > 0) +~~~ + +## See also + +- [Constraints](constraints.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`DEFAULT` constraint](default-value.html) +- [`REFERENCES` constraint (Foreign Key)](foreign-key.html) +- [`NOT NULL` constraint](not-null.html) +- [`PRIMARY KEY` constraint](primary-key.html) +- [`UNIQUE` constraint](unique.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) diff --git a/v20.2/cluster-settings.md b/v20.2/cluster-settings.md new file mode 100644 index 00000000000..e82577a12f7 --- /dev/null +++ b/v20.2/cluster-settings.md @@ -0,0 +1,43 @@ +--- +title: Cluster Settings +summary: Learn about cluster settings that apply to all nodes of a CockroachDB cluster. +toc: false +--- + +Cluster settings apply to all nodes of a CockroachDB cluster and control, for example, whether or not to share diagnostic details with Cockroach Labs as well as advanced options for debugging and cluster tuning. + +They can be updated anytime after a cluster has been started, but only by a member of the `admin` role, to which the `root` user belongs by default. + +{{site.data.alerts.callout_info}} +In contrast to cluster-wide settings, node-level settings apply to a single node. They are defined by flags passed to the `cockroach start` command when starting a node and cannot be changed without stopping and restarting the node. For more details, see [Start a Node](cockroach-start.html). +{{site.data.alerts.end}} + +## Settings + +{{site.data.alerts.callout_danger}} +Many cluster settings are intended for tuning CockroachDB internals. Before changing these settings, we strongly encourage you to discuss your goals with Cockroach Labs; otherwise, you use them at your own risk. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/sql/settings/settings.md %} + +## View current cluster settings + +Use the [`SHOW CLUSTER SETTING`](show-cluster-setting.html) statement. + +## Change a cluster setting + +Use the [`SET CLUSTER SETTING`](set-cluster-setting.html) statement. + +Before changing a cluster setting, please note the following: + +- Changing a cluster setting is not instantaneous, as the change must be propagated to other nodes in the cluster. + +- Do not change cluster settings while [upgrading to a new version of CockroachDB](upgrade-cockroach-version.html). Wait until all nodes have been upgraded before you make the change. + +## See also + +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`SHOW CLUSTER SETTING`](show-cluster-setting.html) +- [Diagnostics Reporting](diagnostics-reporting.html) +- [Start a Node](cockroach-start.html) +- [Use the Built-in SQL Client](cockroach-sql.html) diff --git a/v20.2/cluster-setup-troubleshooting.md b/v20.2/cluster-setup-troubleshooting.md new file mode 100644 index 00000000000..7585a2710cd --- /dev/null +++ b/v20.2/cluster-setup-troubleshooting.md @@ -0,0 +1,511 @@ +--- +title: Troubleshoot Cluster Setup +summary: Learn how to troubleshoot issues with starting CockroachDB clusters +toc: true +--- + +If you're having trouble starting or scaling your cluster, this page will help you troubleshoot the issue. + +To use this guide, it's important to understand some of CockroachDB's terminology: + + - A **Cluster** acts as a single logical database, but is actually made up of many cooperating nodes. + - **Nodes** are single instances of the `cockroach` binary running on a machine. It's possible (though atypical) to have multiple nodes running on a single machine. + +## Cannot run a single-node CockroachDB cluster + +Try running: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start-single-node --insecure --logtostderr +~~~ + +If the process exits prematurely, check for the following: + +### An existing storage directory + +When starting a node, the directory you choose to store the data in also contains metadata identifying the cluster the data came from. This causes conflicts when you've already started a node on the server, have quit `cockroach`, and then tried to start another cluster using the same directory. Because the existing directory's cluster ID doesn't match the new cluster ID, the node cannot start. + +**Solution:** Disassociate the node from the existing directory where you've stored CockroachDB data. For example, you can do either of the following: + +- Choose a different directory to store the CockroachDB data: + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --store= --insecure + ~~~ +- Remove the existing directory and start the node again: + {% include copy-clipboard.html %} + ~~~ shell + $ rm -r cockroach-data/ + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --insecure --logtostderr + ~~~ + +### Toolchain incompatibility + +The components of the toolchain might have some incompatibilities that need to be resolved. For example, a few months ago, there was an incompatibility between Xcode 8.3 and Go 1.8 that caused any Go binaries created with that toolchain combination to crash immediately. + +### Incompatible CPU + +If the `cockroach` process had exit status `132 (SIGILL)`, it attempted to use an instruction that is not supported by your CPU. Non-release builds of CockroachDB may not be able to run on older hardware platforms than the one used to build them. Release builds should run on any x86-64 CPU. + +### Default ports already in use + +Other services may be running on port 26257 or 8080 (CockroachDB's default `--listen-addr` port and `--http-addr` port respectively). You can either stop those services or start your node with different ports, specified in the [`--listen-addr` and `--http-addr` flags](cockroach-start.html#networking). + + If you change the port, you will need to include the `--port=` flag in each subsequent cockroach command or change the `COCKROACH_PORT` environment variable. + +### Single-node networking issues + +Networking issues might prevent the node from communicating with itself on its hostname. You can control the hostname CockroachDB uses with the [`--listen-addr` flag](cockroach-start.html#networking). + + If you change the host, you will need to include `--host=` in each subsequent cockroach command. + +### CockroachDB process hangs when trying to start a node in the background + +See [Why is my process hanging when I try to start it in the background?](operational-faqs.html#why-is-my-process-hanging-when-i-try-to-start-it-in-the-background) + +## Cannot run SQL statements using built-in SQL client + +If the CockroachDB node appeared to [start successfully](start-a-local-cluster.html), in a separate terminal run: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure -e "show databases" +~~~ + +You should see a list of the built-in databases: + +~~~ + database_name ++---------------+ + defaultdb + postgres + system +(3 rows) +~~~ + +If you’re not seeing the output above, check for the following: + +- `connection refused` error, which indicates you have not included some flag that you used to start the node. We have additional troubleshooting steps for this error [here](common-errors.html#connection-refused). +- The node crashed. To ascertain if the node crashed, run `ps | grep cockroach` to look for the `cockroach` process. If you cannot locate the `cockroach` process (i.e., it crashed), [file an issue](file-an-issue.html), including the logs from your node and any errors you received. + +## Cannot run a multi-node CockroachDB cluster on the same machine + +{{site.data.alerts.callout_info}} +Running multiple nodes on a single host is useful for testing out CockroachDB, but it's not recommended for production deployments. To run a physically distributed cluster in production, see [Manual Deployment](manual-deployment.html) or [Orchestrated Deployment](orchestration.html). Also be sure to review the [Production Checklist](recommended-production-settings.html). +{{site.data.alerts.end}} + +If you are trying to run all nodes on the same machine, you might get the following errors: + +### Store directory already exists + +~~~ +ERROR: could not cleanup temporary directories from record file: could not lock temporary directory /Users/amruta/go/src/github.com/cockroachdb/cockroach/cockroach-data/cockroach-temp301343769, may still be in use: IO error: While lock file: /Users/amruta/go/src/github.com/cockroachdb/cockroach/cockroach-data/cockroach-temp301343769/TEMP_DIR.LOCK: Resource temporarily unavailable +~~~ + +**Explanation:** When starting a new node on the same machine, the directory you choose to store the data in also contains metadata identifying the cluster the data came from. This causes conflicts when you've already started a node on the server and then tried to start another cluster using the same directory. + +**Solution:** Choose a different directory to store the CockroachDB data. + +### Port already in use + +~~~ +ERROR: cockroach server exited with error: consider changing the port via --listen-addr: listen tcp 127.0.0.1:26257: bind: address already in use +~~~ + +**Solution:** Change the `--port`, `--http-port` flags for each new node that you want to run on the same machine. + +## Cannot join a node to an existing CockroachDB cluster + +### Store directory already exists + +When joining a node to a cluster, you might receive one of the following errors: + +~~~ +no resolvers found; use --join to specify a connected node + +node belongs to cluster {"cluster hash"} but is attempting to connect to a gossip network for cluster {"another cluster hash"} +~~~ + +**Explanation:** When starting a node, the directory you choose to store the data in also contains metadata identifying the cluster the data came from. This causes conflicts when you've already started a node on the server, have quit the `cockroach` process, and then tried to join another cluster. Because the existing directory's cluster ID doesn't match the new cluster ID, the node cannot join it. + +**Solution:** Disassociate the node from the existing directory where you've stored CockroachDB data. For example, you can do either of the following: + +- Choose a different directory to store the CockroachDB data: + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start --store= --join= + ~~~ +- Remove the existing directory and start a node joining the cluster again: + {% include copy-clipboard.html %} + ~~~ shell + $ rm -r cockroach-data/ + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start --join=:26257 + ~~~ + +### Incorrect `--join` address + +If you try to add another node to the cluster, but the `--join` address is not pointing at any of the existing nodes, then the process will never complete, and you'll see a continuous stream of warnings like this: + +~~~ +W180817 17:01:56.506968 886 vendor/google.golang.org/grpc/clientconn.go:942 Failed to dial localhost:20000: grpc: the connection is closing; please retry. +W180817 17:01:56.510430 914 vendor/google.golang.org/grpc/clientconn.go:1293 grpc: addrConn.createTransport failed to connect to {localhost:20000 0 }. Err :connection error: desc = "transport: Error while dialing dial tcp [::1]:20000: connect: connection refused". Reconnecting… +~~~ + +**Explanation:** These warnings tell you that the node cannot establish a connection with the address specified in the `--join` flag. Without a connection to the cluster, the node cannot join. + +**Solution:** To successfully join the node to the cluster, start the node again, but this time include a correct `--join` address. + +### Missing `--join` address + +If you try to add another node to the cluster, but the `--join` address is missing entirely, then the new node will initialize itself as a new cluster instead of joining the existing cluster. You can see this in the status field printed to stdout: + +~~~ +CockroachDB node starting at 2018-02-08 16:30:26.690638 +0000 UTC (took 0.2s) +build: CCL v2.2.0-alpha.20181217 @ 2018/01/08 17:30:06 (go1.8.3) +admin: https://localhost:8085 +sql: postgresql://root@localhost:26262?sslcert=certs%2Fclient.root.crt&sslkey=certs%2Fclient.root.key&sslmode=verify-full&sslrootcert=certs%2Fca.crt +logs: /Users/jesseseldess/cockroachdb-training/cockroach-v2.2.0-alpha.20181217.darwin-10.9-amd64/node6/logs +store[0]: path=/Users/jesseseldess/cockroachdb-training/cockroach-v2.2.0-alpha.20181217.darwin-10.9-amd64/node6 +status: initialized new cluster +clusterID: cfcd80ee-9005-4975-9ae9-9c36d9aaa57e +nodeID: 1 +~~~ + +If you then stop the node and start it again with a correct `--join` address, the startup process will fail because the cluster will notice that the node's cluster ID does not match the cluster ID of the nodes it is trying to join to: + +~~~ +W180815 17:21:00.316845 237 gossip/client.go:123 [n1] failed to start gossip client to localhost:26258: initial connection heartbeat failed: rpc error: code = Unknown desc = client cluster ID "9a6ed934-50e8-472a-9d55-c6ecf9130984" doesn't match server cluster ID "ab6960bb-bb61-4e6f-9190-992f219102c6" +~~~ + +**Solution:** To successfully join the node to the cluster, you need to remove the node's data directory, which is where its incorrect cluster ID is stored, and start the node again. + +## Client connection issues + +If a client cannot connect to the cluster, check basic network connectivity (`ping`), port connectivity (`telnet`), and certificate validity. + +### Networking issues + +Most networking-related issues are caused by one of two issues: + +- Firewall rules, which require your network administrator to investigate +- Inaccessible hostnames on your nodes, which can be controlled with the `--listen-addr` and `--advertise-addr` flags on [`cockroach start`](cockroach-start.html#networking) + + +**Solution:** + +To check your networking setup: + +1. Use `ping`. Every machine you are using as a CockroachDB node should be able to ping every other machine, using the hostnames or IP addresses used in the `--join` flags (and the `--advertise-host` flag if you are using it). + +2. If the machines are all pingable, check if you can connect to the appropriate ports. With your CockroachDB nodes still running, log in to each node and use `telnet` or` nc` to verify machine to machine connectivity on the desired port. For instance, if you are running CockroachDB on the default port of 26257, run either: + - `telnet 26257` + - `nc 26257` + + Both `telnet` and `nc` will exit immediately if a connection cannot be established. If you are running in a firewalled environment, the firewall might be blocking traffic to the desired ports even though it is letting ping packets through. + +To efficiently troubleshoot the issue, it's important to understand where and why it's occurring. We recommend checking the following network-related issues: + +- By default, CockroachDB advertises itself to other nodes using its hostname. If your environment doesn't support DNS or the hostname is not resolvable, your nodes cannot connect to one another. In these cases, you can: + - Change the hostname each node uses to advertises itself with `--advertise-addr` + - Set `--listen-addr=` if the IP is a valid interface on the machine +- Every node in the cluster should be able to ping each other node on the hostnames or IP addresses you use in the `--join`, `--listen-addr`, or `--advertise-addr` flags. +- Every node should be able to connect to other nodes on the port you're using for CockroachDB (26257 by default) through `telnet` or `nc`: + - `telnet 26257` + - `nc 26257` + +Again, firewalls or hostname issues can cause any of these steps to fail. + +### Network partition + +If the Admin UI lists any dead nodes on the [**Cluster Overview** page](admin-ui-cluster-overview-page.html), then you might have a network partition. + +**Explanation:** A network partition prevents nodes from communicating with each other in one or both directions. This can be due to a configuration problem with the network, such as when whitelisted IP addresses or hostnames change after a node is torn down and rebuilt. In a symmetric partition, node communication is broken in both directions. In an asymmetric partition, node communication works in one direction but not the other. + +The effect of a network partition depends on which nodes are partitioned, where the ranges are located, and to a large extent, whether [localities](cockroach-start.html#locality) are defined. If localities are not defined, a partition that cuts off at least (n-1)/2 nodes will cause data unavailability. + +**Solution:** + +To identify a network partition: + +1. Access the [Network Latency](admin-ui-network-latency-page.html) page of the Admin UI. +2. In the **Latencies** table, check for nodes with [no connections](admin-ui-network-latency-page.html#no-connections). This indicates that a node cannot communicate with another node, and might indicate a network partition. + +## CockroachDB authentication issues + +### Missing certificate + +If you try to add a node to a secure cluster without providing the node's security certificate, you will get the following error: + +~~~ +problem with CA certificate: not found +* +* ERROR: cannot load certificates. +* Check your certificate settings, set --certs-dir, or use --insecure for insecure clusters. +* +* problem with CA certificate: not found +* +Failed running "start" +~~~ + +**Explanation:** The error tells you that because the cluster is secure, it requires the new node to provide its security certificate in order to join. + +**Solution:** To successfully join the node to the cluster, start the node again, but this time include the `--certs-dir` flag + +### Certification expiration + +If you’re running a secure cluster, be sure to monitor your certificate expiration. If one of the inter-node certificates expires, nodes will no longer be able to communicate which can look like a network partition. + +To check the certificate expiration date: + +1. [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui). +2. Click the gear icon on the left-hand navigation bar to access the **Advanced Debugging** page. +3. Scroll down to the **Even More Advanced Debugging** section. Click **All Nodes**. The **Node Diagnostics** page appears. Click the certificates for each node and check the expiration date for each certificate in the Valid Until field. + +### Client password not set + +While connecting to a secure cluster as a user, CockroachDB first checks if the client certificate exists in the `cert` directory. If the client certificate doesn’t exist, it prompts for a password. If password is not set and you press Enter, the connection attempt fails, and the following error is printed to `stderr`: + +~~~ +Error: pq: invalid password +Failed running "sql" +~~~ + +**Solution:** To successfully connect to the cluster, you must first either generate a client certificate or create a password for the user. + +## Clock sync issues + +### Node clocks are not properly synchronized + +See the following FAQs: + +- [What happens when node clocks are not properly synchronized](operational-faqs.html#what-happens-when-node-clocks-are-not-properly-synchronized) +- [How can I tell how well node clocks are synchronized](operational-faqs.html#how-can-i-tell-how-well-node-clocks-are-synchronized) + +## Capacity planning issues + +Following are some of the possible issues you might have while planning capacity for your cluster: + +- Running CPU at close to 100% utilization with high run queue will result in poor performance. +- Running RAM at close to 100% utilization triggers Linux OOM and/or swapping that will result in poor performance or stability issues. +- Running storage at 100% capacity causes writes to fail, which in turn can cause various processes to stop. +- Running storage at 100% utilization read/write will causes poor service time. +- Running network at 100% utilization causes response between databases and client to be poor. + +**Solution:** [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) and navigate to **Metrics > Hardware** dashboard to monitor the following metrics: + +First, check adequate capacity was available for the incident for the following components. + +Type | Time Series | What to look for +--------|--------|--------| +RAM capacity | Memory Usage | Any non-zero value +CPU capacity | CPU Percent | Consistent non-zero values +Network capacity | Network Bytes Received
Network Bytes Sent | Any non-zero value + +Check Near Out of Capacity Conditions: + +Type | Time Series | What to look for +--------|--------|--------| +RAM capacity | Memory Usage | Consistently more than 80% +CPU capacity | CPU Percent | Consistently less than 20% in idle (ie:80% busy) +Network capacity | Network Bytes Received
Network Bytes Sent | Consistently more than 50% capacity for both + +## Storage issues + +### Disks filling up + +Like any database system, if you run out of disk space the system will no longer be able to accept writes. Additionally, a CockroachDB node needs a small amount of disk space (a few GBs to be safe) to perform basic maintenance functionality. For more information about this issue, see: + +- [Why is memory usage increasing despite lack of traffic?](operational-faqs.html#why-is-memory-usage-increasing-despite-lack-of-traffic) +- [Why is disk usage increasing despite lack of writes?](operational-faqs.html#why-is-disk-usage-increasing-despite-lack-of-writes) +- [Can I reduce or disable the storage of timeseries data?](operational-faqs.html#can-i-reduce-or-disable-the-storage-of-timeseries-data) + +## Memory issues + +### Suspected memory leak + +A CockroachDB node will grow to consume all of the memory allocated for its `cache`. The default size for the cache is ¼ of physical memory which can be substantial depending on your machine configuration. This growth will occur even if your cluster is otherwise idle due to the internal metrics that a CockroachDB cluster tracks. See the `--cache` flag in [`cockroach start`](cockroach-start.html#general). + +CockroachDB memory usage has 3 components: + +- **Go allocated memory**: Memory allocated by the Go runtime to support query processing and various caches maintained in Go by CockroachDB. These caches are generally small in comparison to the RocksDB cache size. If Go allocated memory is larger than a few hundred megabytes, something concerning is going on. +- **CGo allocated memory**: Memory allocated by the C/C++ libraries linked into CockroachDB and primarily concerns RocksDB and the RocksDB block cache. This is the “cache” mentioned in the note above. The size of CGo allocated memory is usually very close to the configured cache size. +- **Overhead**: The process resident set size minus Go/CGo allocated memory. + +If Go allocated memory is larger than a few hundred megabytes, you might have encountered a memory leak. Go comes with a built-in heap profiler which is already enabled on your CockroachDB process. See this [excellent blog post](https://blog.golang.org/profiling-go-programs) on profiling Go programs. + +**Solution:** To determine Go/CGo allocated memory: + +1. [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui). + +2. Navigate to **Metrics > Runtime** dashboard, and check the **Memory Usage** graph. + +3. On hovering over the graph, the values for the following metrics are displayed: + + Metric | Description + --------|---- + RSS | Total memory in use by CockroachDB. + Go Allocated | Memory allocated by the Go layer. + Go Total | Total memory managed by the Go layer. + CGo Allocated | Memory allocated by the C layer. + CGo Total | Total memory managed by the C layer. + + - If CGo allocated memory is larger than the configured `cache` size, [file an issue](file-an-issue.html). + - If the resident set size (RSS) minus Go/CGo total memory is larger than 100 megabytes, [file an issue](file-an-issue.html). + +### Node crashes because of insufficient memory + +Often when a node exits without a trace or logging any form of error message, we’ve found that it is the operating system killing it suddenly due to low memory. So if you're seeing node crashes where the logs just end abruptly, it's probably because the node is running out of memory. On most Unix systems, you can verify if the `cockroach` process was killed because the node ran out of memory by running: + +~~~ shell +$ sudo dmesg | grep -iC 3 "cockroach" +~~~ + +If the command returns the following message, then you know the node crashed due to insufficient memory: + +~~~ shell +$ host kernel: Out of Memory: Killed process (cockroach). +~~~ + +To rectify the issue, you can either run the cockroachdb process on another node with sufficient memory, or [reduce the cockroachdb memory usage](cockroach-start.html#flags). + +## Decommissioning issues + +### Decommissioning process hangs indefinitely + +**Explanation:** Before decommissioning a node, you need to make sure other nodes are available to take over the range replicas from the node. If no other nodes are available, the decommission process will hang indefinitely. + +**Solution:** Confirm that there are enough nodes with sufficient storage space to take over the replicas from the node you want to remove. + +### Decommissioned nodes displayed in UI forever + +By design, decommissioned nodes are displayed in the Admin UI forever. We retain the list of decommissioned nodes for the following reasons: + +- Decommissioning is not entirely free, so showing those decommissioned nodes in the UI reminds you of the baggage your cluster will have to carry around forever. +- It also explains to future administrations why your node IDs have gaps (e.g., why the nodes are numbered n1, n2, and n8). + +You can follow the discussion here: [https://github.com/cockroachdb/cockroach/issues/24636](https://github.com/cockroachdb/cockroach/issues/24636) + +## Replication issues + +### Admin UI shows under-replicated/unavailable ranges + +When a CockroachDB node dies (or is partitioned) the under-replicated range count will briefly spike while the system recovers. + +**Explanation:** CockroachDB uses consensus replication and requires a quorum of the replicas to be available in order to allow both writes and reads to the range. The number of failures that can be tolerated is equal to (Replication factor - 1)/2. Thus CockroachDB requires (n-1)/2 nodes to achieve quorum. For example, with 3x replication, one failure can be tolerated; with 5x replication, two failures, and so on. + +- Under-replicated Ranges: When a cluster is first initialized, the few default starting ranges will only have a single replica, but as soon as other nodes are available, they will replicate to them until they've reached their desired replication factor. If a range does not have enough replicas, the range is said to be "under-replicated". + +- Unavailable Ranges: If a majority of a range's replicas are on nodes that are unavailable, then the entire range is unavailable and will be unable to process queries. + +**Solution:** + +To identify under-replicated/unavailable ranges: + +1. [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui). + +2. On the **Cluster Overview** page, check the **Replication Status**. If the **Under-replicated ranges** or **Unavailable ranges** count is non-zero, then you have under-replicated or unavailable ranges in your cluster. + +3. Check for a network partition: Click the gear icon on the left-hand navigation bar to access the **Advanced Debugging** page. On the Advanced Debugging page, click **Network Latency**. In the **Latencies** table, check if any cells are marked as “X”. If yes, it indicates that the nodes cannot communicate with those nodes, and might indicate a network partition. If there's no partition, and there's still no upreplication after 5 mins, then [file an issue](file-an-issue.html). + +**Add nodes to the cluster:** + +On the Admin UI’s Cluster Overview page, check if any nodes are down. If the number of nodes down is less than (n-1)/2, then that is most probably the cause of the under-replicated/unavailable ranges. Add nodes to the cluster such that the cluster has the required number of nodes to replicate ranges properly. + +If you still see under-replicated/unavailable ranges on the Cluster Overview page, investigate further: + +1. [Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) +2. Click the gear icon on the left-hand navigation bar to access the **Advanced Debugging** page. +2. Click **Problem Ranges**. +3. In the **Connections** table, identify the node with the under-replicated/unavailable ranges and click the node ID in the Node column. +4. To view the **Range Report** for a range, click on the range number in the **Under-replicated (or slow)** table or **Unavailable** table. +5. On the Range Report page, scroll down to the **Simulated Allocator Output** section. The table contains an error message which explains the reason for the under-replicated range. Follow the guidance in the message to resolve the issue. If you need help understanding the error or the guidance, [file an issue](file-an-issue.html). Please be sure to include the full range report and error message when you submit the issue. + +## Node liveness issues + +"Node liveness" refers to whether a node in your cluster has been determined to be "dead" or "alive" by the rest of the cluster. This is achieved using checks that ensure that each node connected to the cluster is updating its liveness record. This information is shared with the rest of the cluster using an internal gossip protocol. + +Common reasons for node liveness issues include: + +- Heavy I/O load on the node. Because each node needs to update a liveness record on disk, maxing out disk bandwidth can cause liveness heartbeats to be missed. See also: [Capacity planning issues](#capacity-planning-issues). +- Outright I/O failure due to a disk stall. This will cause node liveness issues for the same reasons as listed above. +- Any [Networking issues](#networking-issues) with the node. + +The [Admin UI][admin_ui] provides several ways to check for node liveness issues in your cluster: + +- [Check node heartbeat latency](#check-node-heartbeat-latency) +- [Check node liveness record last update](#check-node-liveness-record-last-update) +- [Check command commit latency](#check-command-commit-latency) + +{{site.data.alerts.callout_info}} +For more information about how node liveness works, see [the architecture documentation on the replication layer](architecture/replication-layer.html#epoch-based-leases-table-data). +{{site.data.alerts.end}} + +### Check node heartbeat latency + +To check node heartbeat latency: + +1. In the [Admin UI][admin_ui], select the **Metrics** tab from the left-hand side of the page. + +2. From the metrics page, select **Dashboard: Distributed** from the dropdown at the top of the page. + +3. Scroll down the metrics page to find the **Node Heartbeat Latency: 99th percentile** and **Node Heartbeat Latency: 90th percentile** graphs. + +**Expected values for a healthy cluster**: Less than 100ms in addition to the network latency between nodes in the cluster. + +### Check node liveness record last update + +To see when a node last updated its liveness record: + +1. Go to the **Node Diagnostics** page of the [Admin UI][admin_ui], which lives at: + + + +2. On the Node Diagnostics page, you will see a table listing information about the nodes in your cluster. To see when a node last updated its liveness record, check the **Updated at** field at the bottom of that node's column. + +**Expected values for a healthy cluster**: When you load this page, the **Updated at** field should be within 4.5 seconds of the current time. If it's higher than that, you will see errors [in the logs](debug-and-error-logs.html). + +### Check command commit latency + +A good signal of I/O load is the **Command Commit Latency** in the **Storage** section of the dashboards. This dashboard measures how quickly [Raft commands](architecture/replication-layer.html) are being committed by nodes in the cluster. + +To view command commit latency: + +1. In the [Admin UI][admin_ui], select the **Metrics** tab from the left-hand side of the page. + +2. From the Metrics page, select **Dashboard: Storage** from the dropdown at the top of the page. + +3. Scroll down the metrics page to find the **Command Commit Latency: 90th percentile** and **Command Commit Latency: 99th percentile** graphs. + +**Expected values for a healthy cluster**: On SSDs, this should be between 1 and 100 milliseconds. On HDDs, this should be no more than 1 second. Note that we [strongly recommend running CockroachDB on SSDs](recommended-production-settings.html#storage). + +## Check for under-replicated or unavailable data + +To see if any data is under-replicated or unavailable in your cluster, use the `system.replication_stats` report as described in [Replication Reports](query-replication-reports.html). + +## Check for replication zone constraint violations + +To see if any of your cluster's [data placement constraints](configure-replication-zones.html#replication-constraints) are being violated, use the `system.replication_constraint_stats` report as described in [Replication Reports](query-replication-reports.html). + +## Check for critical localities + +To see which of your [localities](cockroach-start.html#locality) (if any) are critical, use the `system.replication_critical_localities` report as described in [Replication Reports](query-replication-reports.html). A locality is "critical" for a range if all of the nodes in that locality becoming [unreachable](#node-liveness-issues) would cause the range to become unavailable. In other words, the locality contains a majority of the range's replicas. + +## Something else? + +If we do not have a solution here, you can try using our other [support resources](support-resources.html), including: + +- [StackOverflow](http://stackoverflow.com/questions/tagged/cockroachdb) +- [CockroachDB Community Forum](https://forum.cockroachlabs.com) +- [Chatting with our developers on Slack](https://cockroachdb.slack.com) + + + +[admin_ui]: admin-ui-access-and-navigate.html#accessing-the-admin-ui-for-a-secure-cluster diff --git a/v20.2/cockroach-cert.md b/v20.2/cockroach-cert.md new file mode 100644 index 00000000000..25276b9b85f --- /dev/null +++ b/v20.2/cockroach-cert.md @@ -0,0 +1,330 @@ +--- +title: cockroach cert +summary: A secure CockroachDB cluster uses TLS for encrypted inter-node and client-node communication. +toc: true +redirect_from: create-security-certificates.html +key: create-security-certificates.html +--- + +To secure your CockroachDB cluster's inter-node and client-node communication, you need to provide a Certificate Authority (CA) certificate that has been used to sign keys and certificates (SSLs) for: + +- Nodes +- Clients +- Admin UI (optional) + +To create these certificates and keys, use the `cockroach cert` [commands](cockroach-commands.html) with the appropriate subcommands and flags, use [`openssl` commands](https://wiki.openssl.org/index.php/), or use a [custom CA](create-security-certificates-custom-ca.html) (for example, a public CA or your organizational CA). + +
+ + + +
+ +{{site.data.alerts.callout_success}}For details about when and how to change security certificates without restarting nodes, see Rotate Security Certificates.{{site.data.alerts.end}} + +## How security certificates work + +1. Using the `cockroach cert` command, you create a CA certificate and key and then node and client certificates that are signed by the CA certificate. Since you need access to a copy of the CA certificate and key to create node and client certs, it's best to create everything in one place. + +2. You then upload the appropriate node certificate and key and the CA certificate to each node, and you upload the appropriate client certificate and key and the CA certificate to each client. + +3. When nodes establish contact to each other, and when clients establish contact to nodes, they use the CA certificate to verify each other's identity. + +## Subcommands + +Subcommand | Usage +-----------|------ +`create-ca` | Create the self-signed certificate authority (CA), which you'll use to create and authenticate certificates for your entire cluster. +`create-node` | Create a certificate and key for a specific node in the cluster. You specify all addresses at which the node can be reached and pass appropriate flags. +`create-client` | Create a certificate and key for a [specific user](create-user.html) accessing the cluster from a client. You specify the username of the user who will use the certificate and pass appropriate flags. +`list` | List certificates and keys found in the certificate directory. + +## Certificate directory + +When using `cockroach cert` to create node and client certificates, you will need access to a local copy of the CA certificate and key. It is therefore recommended to create all certificates and keys in one place and then distribute node and client certificates and keys appropriately. For the CA key, be sure to store it somewhere safe and keep a backup; if you lose it, you will not be able to add new nodes or clients to your cluster. For a walkthrough of this process, see [Manual Deployment](manual-deployment.html). + +## Required keys and certificates + +The `create-*` subcommands generate the CA certificate and all node and client certificates and keys in a single directory specified by the `--certs-dir` flag, with the files named as follows: + +### Node key and certificates + +File name pattern | File usage +-------------|------------ +`ca.crt` | CA certificate. +`node.crt` | Server certificate.

`node.crt` must be signed by `ca.crt` and must have `CN=node` and the list of IP addresses and DNS names listed in `Subject Alternative Name` field. CockroachDB also supports [wildcard notation in DNS names](https://en.wikipedia.org/wiki/Wildcard_certificate). +`node.key` | Key for server certificate. + +### Client key and certificates + +File name pattern | File usage +-------------|------------ +`ca.crt` | CA certificate. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Must be signed by `ca.crt`. Also, `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`) +`client..key` | Key for the client certificate. + +Optionally, if you have a certificate issued by a public CA to securely access the Admin UI, you need to place the certificate and key (`ui.crt` and `ui.key` respectively) in the directory specified by the `--certs-dir` flag. For more information, refer to [Use a UI certificate and key to access the Admin UI](create-security-certificates-custom-ca.html#accessing-the-admin-ui-for-a-secure-cluster). + +Note the following: + +- By default, the `node.crt` is multi-functional, as in the same certificate is used for both incoming connections (from SQL and Admin UI clients, and from other CockroachDB nodes) and for outgoing connections to other CockroachDB nodes. To make this possible, the `node.crt` created using the `cockroach cert` command has `CN=node` and the list of IP addresses and DNS names listed in `Subject Alternative Name` field. + +- The CA key is never loaded automatically by `cockroach` commands, so it should be created in a separate directory, identified by the `--ca-key` flag. + +- Keys (files ending in `.key`) must not have group or world permissions (maximum permissions are 0700, or `rwx------`). This check can be disabled by setting the environment variable `COCKROACH_SKIP_KEY_PERMISSION_CHECK=true`. + +## Synopsis + +Create the CA certificate and key: + +~~~ shell +$ cockroach cert create-ca \ + --certs-dir=[path-to-certs-directory] \ + --ca-key=[path-to-ca-key] +~~~ + +Create a node certificate and key: + +~~~ shell +$ cockroach cert create-node \ + [node-hostname] \ + [node-other-hostname] \ + [node-yet-another-hostname] \ + [hostname-in-wildcard-notation] \ + --certs-dir=[path-to-certs-directory] \ + --ca-key=[path-to-ca-key] +~~~ + +Create a client certificate and key: + +~~~ shell +$ cockroach cert create-client \ + [username] \ + --certs-dir=[path-to-certs-directory] \ + --ca-key=[path-to-ca-key] +~~~ + +List certificates and keys: + +~~~ shell +$ cockroach cert list \ + --certs-dir=[path-to-certs-directory] +~~~ + +View help: + +~~~ shell +$ cockroach cert --help +~~~ +~~~ shell +$ cockroach cert --help +~~~ + +## Flags + +The `cert` command and subcommands support the following [general-use](#general) and [logging](#logging) flags. + +### General + +Flag | Description +-----|----------- +`--certs-dir` | The path to the [certificate directory](#certificate-directory) containing all certificates and keys needed by `cockroach` commands.

This flag is used by all subcommands.

**Default:** `${HOME}/.cockroach-certs/` +`--ca-key` | The path to the private key protecting the CA certificate.

This flag is required for all `create-*` subcommands. When used with `create-ca` in particular, it defines where to create the CA key; the specified directory must exist.

**Env Variable:** `COCKROACH_CA_KEY` +`--allow-ca-key-reuse` | When running the `create-ca` subcommand, pass this flag to re-use an existing CA key identified by `--ca-key`. Otherwise, a new CA key will be generated.

This flag is used only by the `create-ca` subcommand. It helps avoid accidentally re-using an existing CA key. +`--overwrite` | When running `create-*` subcommands, pass this flag to allow existing files in the certificate directory (`--certs-dir`) to be overwritten.

This flag helps avoid accidentally overwriting sensitive certificates and keys. +`--lifetime` | The lifetime of the certificate, in hours, minutes, and seconds.

Certificates are valid from the time they are created through the duration specified in `--lifetime`.

**Default:** `87840h0m0s` (10 years) +`--key-size` | The size of the CA, node, or client key, in bits.

**Default:** `2048` + `--also-generate-pkcs8-key` | Also create a key in [PKCS#8 format](https://tools.ietf.org/html/rfc5208), which is the standard key encoding format used by Java. For example usage, see [Build a Java App with CockroachDB](build-a-java-app-with-cockroachdb.html). + +### Logging + +By default, the `cert` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +### Create the CA certificate and key pair + +1. Create two directories: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir my-safe-directory + ~~~ + - `certs`: You'll generate your CA certificate and all node and client certificates and keys in this directory and then upload some of the files to your nodes. + - `my-safe-directory`: You'll generate your CA key in this directory and then reference the key when generating node and client certificates. After that, you'll keep the key safe and secret; you will not upload it to your nodes. + +2. Generate the CA certificate and key: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ls -l certs + ~~~ + + ~~~ + total 8 + -rw-r--r-- 1 maxroach maxroach 1.1K Jul 10 14:12 ca.crt + ~~~ + +### Create the certificate and key pairs for nodes + +1. Generate the certificate and key for the first node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + node1.example.com \ + node1.another-example.com \ + *.dev.another-example.com \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ls -l certs + ~~~ + + ~~~ + total 24 + -rw-r--r-- 1 maxroach maxroach 1.1K Jul 10 14:12 ca.crt + -rw-r--r-- 1 maxroach maxroach 1.2K Jul 10 14:16 node.crt + -rw------- 1 maxroach maxroach 1.6K Jul 10 14:16 node.key + ~~~ + +2. Upload certificates to the first node: + + {% include copy-clipboard.html %} + ~~~ shell + # Create the certs directory: + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + # Upload the CA certificate and node certificate and key: + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + +3. Delete the local copy of the first node's certificate and key: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm certs/node.crt certs/node.key + ~~~ + + {{site.data.alerts.callout_info}}This is necessary because the certificates and keys for additional nodes will also be named node.crt and node.key As an alternative to deleting these files, you can run the next cockroach cert create-node commands with the --overwrite flag.{{site.data.alerts.end}} + +4. Create the certificate and key for the second node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + node2.example.com \ + node2.another-example.com \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ ls -l certs + ~~~ + + ~~~ + total 24 + -rw-r--r-- 1 maxroach maxroach 1.1K Jul 10 14:12 ca.crt + -rw-r--r-- 1 maxroach maxroach 1.2K Jul 10 14:17 node.crt + -rw------- 1 maxroach maxroach 1.6K Jul 10 14:17 node.key + ~~~ + +5. Upload certificates to the second node: + + {% include copy-clipboard.html %} + ~~~ shell + # Create the certs directory: + $ ssh @ "mkdir certs" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + # Upload the CA certificate and node certificate and key: + $ scp certs/ca.crt \ + certs/node.crt \ + certs/node.key \ + @:~/certs + ~~~ + +6. Repeat steps 3 - 5 for each additional node. + +### Create the certificate and key pair for a client + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert create-client \ +maxroach \ +--certs-dir=certs \ +--ca-key=my-safe-directory/ca.key +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ ls -l certs +~~~ + +~~~ +total 40 +-rw-r--r-- 1 maxroach maxroach 1.1K Jul 10 14:12 ca.crt +-rw-r--r-- 1 maxroach maxroach 1.1K Jul 10 14:13 client.maxroach.crt +-rw------- 1 maxroach maxroach 1.6K Jul 10 14:13 client.maxroach.key +-rw-r--r-- 1 maxroach maxroach 1.2K Jul 10 14:17 node.crt +-rw------- 1 maxroach maxroach 1.6K Jul 10 14:17 node.key +~~~ + +### List certificates and keys + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach cert list \ +--certs-dir=certs +~~~ + +~~~ +Certificate directory: certs ++-----------------------+---------------------+---------------------+------------+--------------------------------------------------------+-------+ +| Usage | Certificate File | Key File | Expires | Notes | Error | ++-----------------------+---------------------+---------------------+------------+--------------------------------------------------------+-------+ +| Certificate Authority | ca.crt | | 2027/07/18 | num certs: 1 | | +| Node | node.crt | node.key | 2022/07/14 | addresses: node2.example.com,node2.another-example.com | | +| Client | client.maxroach.crt | client.maxroach.key | 2022/07/14 | user: maxroach | | ++-----------------------+---------------------+---------------------+------------+--------------------------------------------------------+-------+ +(3 rows) +~~~ + +## See also + +- [Security overview](security-overview.html) +- [Authentication](authentication.html) +- [Client Connection Parameters](connection-parameters.html) +- [Rotate Security Certificates](rotate-certificates.html) +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Local Deployment](secure-a-cluster.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-commands.md b/v20.2/cockroach-commands.md new file mode 100644 index 00000000000..a4325936eb6 --- /dev/null +++ b/v20.2/cockroach-commands.md @@ -0,0 +1,50 @@ +--- +title: Cockroach Commands +summary: Learn the commands for configuring, starting, and managing a CockroachDB cluster. +toc: true +--- + +This page introduces the `cockroach` commands for configuring, starting, and managing a CockroachDB cluster, as well as environment variables that can be used in place of certain flags. + +You can run `cockroach help` in your shell to get similar guidance. + + +## Commands + +Command | Usage +--------|---- +[`cockroach start`](cockroach-start.html) | Start a node as part of a multi-node cluster. +[`cockroach init`](cockroach-init.html) | Initialize a multi-node cluster. +[`cockroach start-single-node`](cockroach-start-single-node.html) | Start a single-node cluster. +[`cockroach cert`](cockroach-cert.html) | Create CA, node, and client certificates. +[`cockroach quit`](cockroach-quit.html) | Temporarily stop a node or permanently remove a node. +[`cockroach sql`](cockroach-sql.html) | Use the built-in SQL client. +[`cockroach sqlfmt`](cockroach-sqlfmt.html) | Reformat SQL queries for enhanced clarity. +[`cockroach node`](cockroach-node.html) | List node IDs, show their status, decommission nodes for removal, or recommission nodes. +[`cockroach dump`](cockroach-dump.html) | Back up a table by outputting the SQL statements required to recreate the table and all its rows. +[`cockroach demo`](cockroach-demo.html) | Start a temporary, in-memory CockroachDB cluster, and open an interactive SQL shell to it. +[`cockroach gen`](cockroach-gen.html) | Generate manpages, a bash completion file, example SQL data, or an HAProxy configuration file for a running cluster. +[`cockroach version`](cockroach-version.html) | Output CockroachDB version details. +[`cockroach debug ballast`](cockroach-debug-ballast.html) | Create a large, unused file in a node's storage directory that you can delete if the node runs out of disk space. +[`cockroach debug encryption-active-key`](cockroach-debug-encryption-active-key.html) | View the encryption algorithm and store key. +[`cockroach debug zip`](cockroach-debug-zip.html) | Generate a `.zip` file that can help Cockroach Labs troubleshoot issues with your cluster. +[`cockroach debug merge-logs`](cockroach-debug-merge-logs.html) | Merge multiple log files from different machines into a single stream. +[`cockroach workload`](cockroach-workload.html) | Run a built-in load generator against a cluster. +[`cockroach nodelocal upload`](cockroach-nodelocal-upload.html) | Upload a file to the `externalIODir` on a node's local file system. + +## Environment variables + +For many common `cockroach` flags, such as `--port` and `--user`, you can set environment variables once instead of manually passing the flags each time you execute commands. + +- To find out which flags support environment variables, see the documentation for each [command](#commands). +- To output the current configuration of CockroachDB and other environment variables, run `env`. +- When a node uses environment variables on [startup](cockroach-start.html), the variable names are printed to the node's logs; however, the variable values are not. + +CockroachDB prioritizes command flags, environment variables, and defaults as follows: + +1. If a flag is set for a command, CockroachDB uses it. +2. If a flag is not set for a command, CockroachDB uses the corresponding environment variable. +3. If neither the flag nor environment variable is set, CockroachDB uses the default for the flag. +4. If there's no flag default, CockroachDB gives an error. + +For more details, see [Client Connection Parameters](connection-parameters.html). diff --git a/v20.2/cockroach-debug-ballast.md b/v20.2/cockroach-debug-ballast.md new file mode 100644 index 00000000000..37a1606b9b4 --- /dev/null +++ b/v20.2/cockroach-debug-ballast.md @@ -0,0 +1,58 @@ +--- +title: cockroach debug ballast +summary: Create a large, unused file in a node's storage directory that you can delete if the node runs out of disk space. +toc: true +redirect_from: cockroach-debug-ballast.html +key: cockroach-debug-ballast.html +--- + +The `cockroach debug ballast` [command](cockroach-commands.html) creates a large, unused file that you can place in a node's storage directory. In the case that a node runs out of disk space and shuts down, you can delete the ballast file to free up enough space to be able to restart the node. + +- In addition to placing a ballast file in each node's storage directory, it is important to actively [monitor remaining disk space](monitoring-and-alerting.html#events-to-alert-on). +- Ballast files may be created in many ways, including the standard `dd` command. `cockroach debug ballast` uses the `fallocate` system call when available, so it will be faster than `dd`. + +## Subcommands + +{% include {{ page.version.version }}/misc/debug-subcommands.md %} + +## Synopsis + +Create a ballast file: + +~~~ shell +$ cockroach debug ballast [path to ballast file] [flags] +~~~ + +View help: + +~~~ shell +$ cockroach debug ballast --help +~~~ + +## Flags + +Flag | Description +-----|----------- +`--size`
`-z` | The amount of space to fill, or to leave available, in a node's storage directory via a ballast file. Positive values equal the size of the ballast file. Negative values equal the amount of space to leave after creating the ballast file. This can be a percentage (notated as a decimal or with %) or any bytes-based unit, for example:

`--size=1000000000 ----> 1000000000 bytes`
`--size=1GiB ----> 1073741824 bytes`
`--size=5% ----> 5% of available space`
`--size=0.05 ----> 5% of available space`
`--size=.05 ----> 5% of available space`

**Default:** `1GB` + +## Examples + +### Create a 1GB ballast file (default) + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug ballast cockroach-data/ballast.txt +~~~ + +### Create a ballast file of a different size + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug ballast cockroach-data/ballast.txt --size=2GB +~~~ + +## See also + +- [Other Cockroach Commands](cockroach-commands.html) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Production Checklist](recommended-production-settings.html) diff --git a/v20.2/cockroach-debug-encryption-active-key.md b/v20.2/cockroach-debug-encryption-active-key.md new file mode 100644 index 00000000000..4cc3ff54a4b --- /dev/null +++ b/v20.2/cockroach-debug-encryption-active-key.md @@ -0,0 +1,45 @@ +--- +title: cockroach debug encryption-active-key +summary: Learn the command for viewing the algorithm and store key for an encrypted store. +toc: true +redirect_from: debug-encryption-active-key.html +key: debug-encryption-active-key.html +--- + +The `cockroach debug encryption-active-key` [command](cockroach-commands.html) displays the encryption algorithm and store key for an encrypted store. + +## Synopsis + +~~~ shell +$ cockroach debug encryption-active-key [path specified by the store flag] +~~~ + +## Subcommands + +{% include {{ page.version.version }}/misc/debug-subcommands.md %} + +## Example + +Start a node with encryption-at-rest enabled: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --store=cockroach-data --enterprise-encryption=path=cockroach-data,key=aes-128.key,old-key=plain --insecure --certs-dir=certs +~~~ + +View the encryption algorithm and store key: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug encryption-active-key cockroach-data +~~~ + +~~~ +AES128_CTR:be235c29239aa84a48e5e1874d76aebf7fb3c1bdc438cec2eb98de82f06a57a0 +~~~ + +## See also + +- [File an Issue](file-an-issue.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [Troubleshooting Overview](troubleshooting-overview.html) diff --git a/v20.2/cockroach-debug-merge-logs.md b/v20.2/cockroach-debug-merge-logs.md new file mode 100644 index 00000000000..c5d302ad4b4 --- /dev/null +++ b/v20.2/cockroach-debug-merge-logs.md @@ -0,0 +1,76 @@ +--- +title: cockroach debug merge-logs +summary: Learn the command for merging the collected debug logs from all nodes in your cluster. +toc: true +redirect_from: debug-merge-logs.html +key: debug-merge-logs.html +--- + +The `cockroach debug merge-logs` [command](cockroach-commands.html) merges log files from multiple nodes into a single time-ordered stream of messages with an added per-message prefix to indicate the corresponding node. You can use it in conjunction with logs collected using the [`debug zip`](https://www.cockroachlabs.com/docs/stable/cockroach-debug-zip.html) command to aid in debugging. + +{{site.data.alerts.callout_danger}} +The file produced by `cockroach debug merge-log` can contain highly sensitive, unanonymized information, such as usernames, passwords, and possibly your table's data. You should share this data only with Cockroach Labs developers and only after determining the most secure method of delivery. +{{site.data.alerts.end}} + +## Subcommands + +{% include {{ page.version.version }}/misc/debug-subcommands.md %} + +## Synopsis + +~~~ shell +$ cockroach debug merge-logs [log file directory] [flags] +~~~ + +## Flags + +Use the following flags to filter the `debug merge-logs` results for a specified regular expression or time range. + +Flag | Description +-----|----------- +`--filter` | Limit the results to the specified regular expression +`--from` | Start time for the time range filter. +`--to` | End time for the time range filter. + +## Example + +Generate a debug zip file: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug zip ./cockroach-data/logs/debug.zip --insecure +~~~ + +Unzip the file: + +{% include copy-clipboard.html %} +~~~ shell +$ unzip ./cockroach-data/logs/debug.zip +~~~ + +Merge the logs in the debug folder: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug merge-logs debug/nodes/*/logs/* +~~~ + +Alternatively, filter the merged logs for a specified time range: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach debug merge-logs debug/nodes/*/logs/* --from= "18:36:28.208553" --to= "18:36:29.232864" +~~~ + +You can also filter the merged logs for a regular expression: + +{% include copy-clipboard.html %} +~~~ shell +cockroach debug merge-logs debug/nodes/*/logs/* --filter="RUNNING IN INSECURE MODE" +~~~ + +## See also + +- [File an Issue](file-an-issue.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [Troubleshooting Overview](troubleshooting-overview.html) diff --git a/v20.2/cockroach-debug-zip.md b/v20.2/cockroach-debug-zip.md new file mode 100644 index 00000000000..2fa920efb5c --- /dev/null +++ b/v20.2/cockroach-debug-zip.md @@ -0,0 +1,115 @@ +--- +title: cockroach debug zip +summary: Learn the commands for collecting debug information from all nodes in your cluster. +toc: true +redirect_from: debug-zip.html +key: debug-zip.html +--- + +The `cockroach debug zip` [command](cockroach-commands.html) connects to your cluster and gathers information from each active node into a single file (inactive nodes are not included): + +- [Log files](debug-and-error-logs.html) +- Secondary log files (e.g., RocksDB logs, [execution logs](query-behavior-troubleshooting.html#cluster-wide-execution-logs), [slow query logs](query-behavior-troubleshooting.html#using-the-slow-query-log)) +- Cluster events +- Schema change events +- Node liveness +- Gossip data +- Stack traces +- Range lists +- A list of databases and tables +- Jobs +- [Cluster Settings](cluster-settings.html) +- [Metrics](admin-ui-custom-chart-debug-page.html#available-metrics) +- Alerts +- Heap profiles +- Problem ranges +- Sessions +- Queries +- Thread stack traces (Linux only) + +Additionally, you can run the [`debug merge-logs`](cockroach-debug-merge-logs.html) command to merge the collected logs in one file, making it easier to parse them to locate an issue with your cluster. + +{{site.data.alerts.callout_danger}} +The file produced by `cockroach debug zip` can contain highly sensitive, unanonymized information, such as usernames, hashed passwords, and possibly your table's data. You should share this data only with Cockroach Labs developers and only after determining the most secure method of delivery. +{{site.data.alerts.end}} + +## Details + +### Use cases + +There are two scenarios in which `debug zip` is useful: + +- To collect all of your nodes' logs, which you can then parse to locate issues. It's important to note, though, that `debug zip` can only access logs from active nodes. See more information [on this page](#collecting-log-files). + +- If you experience severe or difficult-to-reproduce issues with your cluster, Cockroach Labs might ask you to send us your cluster's debugging information using `cockroach debug zip`. + +### Collecting log files + +When you issue the `debug zip` command, the node that receives the request connects to each other node in the cluster. Once it's connected, the node requests the content of all log files stored on the node, the location of which is determined by the `--log-dir` value when you [started the node](cockroach-start.html). + +Because `debug zip` relies on CockroachDB's distributed architecture, this means that nodes not currently connected to the cluster cannot respond to the request, so their log files *are not* included. In such situations, we recommend using the [`--host` flag](#general) to point `debug zip` at individual nodes until data has been gathered for the entire cluster. + +After receiving the log files from all of the active nodes, the requesting node aggregates the files and writes them to an archive file you specify. + +You can locate logs in the unarchived file's `debug/nodes/[node dir]/logs` directories. + +## Subcommands + +{% include {{ page.version.version }}/misc/debug-subcommands.md %} + +## Synopsis + +~~~ shell +$ cockroach debug zip [ZIP file destination] [flags] +~~~ + +It's important to understand that the `[flags]` here are used to connect to CockroachDB nodes. This means the values you use in those flags must connect to an active node. If no nodes are live, you must [start at least one node](cockroach-start.html). + +## Flags + +The `debug zip` subcommand supports the following [general-use](#general), [client connection](#client-connection), and [logging](#logging) flags. + +### General + +Flag | Description +-----|----------- +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html). The directory must contain valid certificates if running in secure mode.

**Env Variable:** `COCKROACH_CERTS_DIR`
**Default:** `${HOME}/.cockroach-certs/` +`--host` | The server host to connect to. This can be the address of any node in the cluster.

**Env Variable:** `COCKROACH_HOST`
**Default:** `localhost` +`--insecure` | Run in insecure mode. If this flag is not set, the `--certs-dir` flag must point to valid certificates.

**Env Variable:** `COCKROACH_INSECURE`
**Default:** `false` +`--port`
`-p` | The server port to connect to.

**Env Variable:** `COCKROACH_PORT`
**Default:** `26257` + +### Client connection + +Flag | Description +-----|----------- +`--url` | A [connection URL](connection-parameters.html#connect-using-a-url) to use instead of the other arguments.

**Env Variable:** `COCKROACH_URL`
**Default:** no URL + +### Logging + +By default, the `debug zip` command logs errors it experiences to `stderr`. Note that these are errors executing `debug zip`; these are not errors that the logs collected by `debug zip` contain. + +If you need to troubleshoot this command's behavior, you can also change its [logging behavior](debug-and-error-logs.html). + +## Examples + +### Generate a debug zip file + +{% include copy-clipboard.html %} +~~~ shell +# Generate the debug zip file for an insecure cluster: +$ cockroach debug zip ./cockroach-data/logs/debug.zip --insecure --host=200.100.50.25 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +# Generate the debug zip file for a secure cluster: +$ cockroach debug zip ./cockroach-data/logs/debug.zip --host=200.100.50.25 +~~~ + +{{site.data.alerts.callout_info}}Secure examples assume you have the appropriate certificates in the default certificate directory, ${HOME}/.cockroach-certs/.{{site.data.alerts.end}} + +## See also + +- [File an Issue](file-an-issue.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [Troubleshooting Overview](troubleshooting-overview.html) diff --git a/v20.2/cockroach-demo.md b/v20.2/cockroach-demo.md new file mode 100644 index 00000000000..7acb18f2be8 --- /dev/null +++ b/v20.2/cockroach-demo.md @@ -0,0 +1,377 @@ +--- +title: cockroach demo +summary: Use cockroach demo to open a SQL shell to a temporary, in-memory, single-node CockroachDB cluster. +toc: true +--- + +The `cockroach demo` [command](cockroach-commands.html) starts a temporary, in-memory CockroachDB cluster, a preloaded dataset, and opens an [interactive SQL shell](cockroach-sql.html) to the cluster. All [SQL shell commands, client-side options, help, and shortcuts](cockroach-sql.html#sql-shell) supported by the `cockroach sql` command are also supported by the `cockroach demo` command. + +The in-memory cluster persists only as long as the SQL shell is open. As soon as the shell is exited, the cluster and all its data are permanently destroyed. This command is therefore recommended only as an easy way to experiment with the CockroachDB SQL dialect. + +Each instance of `cockroach demo` loads a temporary [enterprise license](https://www.cockroachlabs.com/get-cockroachdb) that expires after an hour. To prevent the loading of a temporary license, set the `--disable-demo-license` flag. + +## Synopsis + +Start an in-memory cluster and open an interactive SQL shell: + +~~~ shell +$ cockroach demo +~~~ + +Start an in-memory cluster with a preloaded dataset and open an interactive SQL shell: + +~~~ shell +$ cockroach demo +~~~ + +Start an in-memory cluster in secure mode and open an interactive SQL shell: + +~~~ shell +$ cockroach demo --insecure=false +~~~ + +Execute SQL from the command line against an in-memory cluster: + +~~~ shell +$ cockroach demo --execute=";" --execute="" +~~~ + +Exit the interactive SQL shell and stop the in-memory cluster: + +~~~ shell +$ \q +ctrl-d +~~~ + +View help: + +~~~ shell +$ cockroach demo --help +~~~ + +## Datasets + +{{site.data.alerts.callout_success}} +To start a demo cluster without a pre-loaded dataset, pass the `--empty` flag. +{{site.data.alerts.end}} + +Workload | Description +---------|------------ +`bank` | A `bank` database, with one `bank` table containing account details. +`intro` | An `intro` database, with one table, `mytable`, with a hidden message. +`kv` | A `kv` database, with one key-value-style table. +`movr` | A `movr` database, with several tables of data for the [MovR example application](movr.html).

By default, `cockroach demo` loads the `movr` database as the [current database](sql-name-resolution.html#current-database), with sample region (`region`) and availability zone (`az`) replica localities for each node specified with the [`--nodes` flag](cockroach-demo.html#flags). +`startrek` | A `startrek` database, with two tables, `episodes` and `quotes`. +`tpcc` | A `tpcc` database, with a rich schema of multiple tables. +`ycsb` | A `ycsb` database, with a `usertable` from the Yahoo! Cloud Serving Benchmark. + +## Flags + +The `demo` command supports the following general-use flags. + +Flag | Description +-----|------------ +`--cache` | For each demo node, the total size for caches. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--cache=.25`
`--cache=25%`
`--cache=1000000000 ----> 1000000000 bytes`
`--cache=1GB ----> 1000000000 bytes`
`--cache=1GiB ----> 1073741824 bytes`

**Default:** `64MiB` +`--demo-locality` | Specify [locality](cockroach-start.html#locality) information for each demo node. The input is a colon-separated list of key-value pairs, where the ith pair is the locality setting for the ith demo cockroach node.

For example, the following option assigns node 1's region to `us-east1` and availability zone to `1`, node 2's region to `us-east2` and availability zone to `2`, and node 3's region to `us-east3` and availability zone to `3`:

`--demo-locality=region=us-east1,az=1:region=us-east1,az=2:region=us-east1,az=3`

By default, `cockroach demo` uses sample region (`region`) and availability zone (`az`) replica localities for each node specified with the `--nodes` flag. +`--disable-demo-license` | Start the demo cluster without loading a temporary [enterprise license](https://www.cockroachlabs.com/get-cockroachdb) that expires after an hour.

Setting the `COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING` environment variable will also prevent the loading of a temporary license, along with preventing the sharing of anonymized [diagnostic details](diagnostics-reporting.html) with Cockroach Labs. +`--echo-sql` | Reveal the SQL statements sent implicitly by the command-line utility. This can also be enabled within the interactive SQL shell via the `\set echo` [shell command](cockroach-sql.html#commands). +`--empty` | Start the demo cluster without a pre-loaded dataset. +`--execute`
`-e` | Execute SQL statements directly from the command line, without opening a shell. This flag can be set multiple times, and each instance can contain one or more statements separated by semi-colons.

If an error occurs in any statement, the command exits with a non-zero status code and further statements are not executed. The results of each statement are printed to the standard output (see `--format` for formatting options). +`--format` | How to display table rows printed to the standard output. Possible values: `tsv`, `csv`, `table`, `raw`, `records`, `sql`, `html`.

**Default:** `table` for sessions that [output on a terminal](cockroach-sql.html#session-and-output-types); `tsv` otherwise

This flag corresponds to the `display_format` [client-side option](cockroach-sql.html#client-side-options) for use in interactive sessions. +`--geo-partitioned-replicas` | Start a 9-node demo cluster with the [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) topology pattern applied to the [`movr`](movr.html) database. +`--insecure` | Set this to `false` to start the demo cluster in secure mode using TLS certificates to encrypt network communication. `--insecure=false` gives you an easy way test out CockroachDB [authorization features](authorization.html) and also creates a password (`admin`) for the `root` user for logging into the Admin UI.

**Env Variable:** `COCKROACH_INSECURE`
**Default:** `false` +`--max-sql-memory` | For each demo node, the maximum in-memory storage capacity for temporary SQL data, including prepared queries and intermediate data rows during query execution. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--max-sql-memory=.25`
`--max-sql-memory=25%`
`--max-sql-memory=10000000000 ----> 1000000000 bytes`
`--max-sql-memory=1GB ----> 1000000000 bytes`
`--max-sql-memory=1GiB ----> 1073741824 bytes`

**Default:** `128MiB` +`--nodes` | Specify the number of in-memory nodes to create for the demo.

**Default:** 1 +`--safe-updates` | Disallow potentially unsafe SQL statements, including `DELETE` without a `WHERE` clause, `UPDATE` without a `WHERE` clause, and `ALTER TABLE ... DROP COLUMN`.

**Default:** `true` for [interactive sessions](cockroach-sql.html#session-and-output-types); `false` otherwise

Potentially unsafe SQL statements can also be allowed/disallowed for an entire session via the `sql_safe_updates` [session variable](set-vars.html). +`--set` | Set a [client-side option](cockroach-sql.html#client-side-options) before starting the SQL shell or executing SQL statements from the command line via `--execute`. This flag may be specified multiple times, once per option.

After starting the SQL shell, the `\set` and `unset` commands can be use to enable and disable client-side options as well. +`--with-load` | Run a demo [`movr`](movr.html) workload against the preloaded `movr` database.

When running a multi-node demo cluster, load is balanced across all nodes. + +## Logging + +By default, the `demo` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Connecting to the demo cluster + +When the SQL shell connects to the demo cluster at startup, it prints a welcome text with some tips and cluster details. Most of these details resemble the [welcome text](cockroach-sql.html#welcome-message) that is printed when connecting `cockroach sql` to a permanent cluster. `cockroach demo` also includes some URLs to connect to the [Admin UI](admin-ui-overview.html) with a web browser, or directly to the cluster with a URL [connection parameter](connection-parameters.html) across a [Unix domain socket connection](cockroach-sql.html#connect-to-a-cluster-listening-for-unix-domain-socket-connections) or a standard TCP connection. + +~~~ shell +# +# Welcome to the CockroachDB demo database! +# +... +# +# Connection parameters: +# (console) http://127.0.0.1:53538 +# (sql) postgres://root:admin@?host=%2Fvar%2Ffolders%2Fpg%2FT%2Fdemo282557495&port=26257 +# (sql/tcp) postgres://root:admin@127.0.0.1:53540?sslmode=require +# +# +# The user "root" with password "admin" has been created. Use it to access the Web UI! +# +... +~~~ + + To return the client connection URLs for all nodes in a demo cluster from within the SQL shell, use the client-side `\demo ls` command: + +{% include copy-clipboard.html %} +~~~ sql +> \demo ls +~~~ + +~~~ +node 1: + (console) http://127.0.0.1:53538 + (sql) postgres://root:admin@?host=%2Fvar%2Ffolders%2Fpg%2FT%2Fdemo282557495&port=26257 + (sql/tcp) postgres://root:admin@127.0.0.1:53540?sslmode=require + +node 2: + (console) http://127.0.0.1:53783 + (sql) postgres://root:admin@?host=%2Fvar%2Ffolders%2Fpg%2FT%2Fdemo282557495&port=26258 + (sql/tcp) postgres://root:admin@127.0.0.1:53785?sslmode=require + +node 3: + (console) http://127.0.0.1:53789 + (sql) postgres://root:admin@?host=%2Fvar%2Ffolders%2Fpg%2FT%2Fdemo282557495&port=26259 + (sql/tcp) postgres://root:admin@127.0.0.1:53791?sslmode=require + +... +~~~ + +{{site.data.alerts.callout_info}} +The `\demo ls` command is **experimental feature**. The interface and output are subject to change. +{{site.data.alerts.end}} + +### Admin UI + +`cockroach demo` serves a local [Admin UI](admin-ui-overview.html) at the **console** link. For the duration of the cluster, you can navigate to this link to monitor the cluster's activity in the Admin UI. To login, you can use the `root` user with password `admin`. + +### URL connection parameters + +You can connect to the demo cluster using a URL connection parameter (e.g., with the [`cockroach sql --url`](cockroach-sql.html#client-connection) command). To establish a [Unix domain socket connection](cockroach-sql.html#connect-to-a-cluster-listening-for-unix-domain-socket-connections) with a client that is installed on the same machine, use the **sql** URL . For standard TCP connections, use the **sql/tcp** URL. + +{{site.data.alerts.callout_info}} +You do not need to create or specify node and client certificates in the connection URL to a secure demo cluster. +{{site.data.alerts.end}} + +## Diagnostics reporting + +By default, `cockroach demo` shares anonymous usage details with Cockroach Labs. To opt out, set the [`diagnostics.reporting.enabled`](diagnostics-reporting.html#after-cluster-initialization) [cluster setting](cluster-settings.html) to `false`. You can also opt out by setting the [`COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING`](diagnostics-reporting.html#at-cluster-initialization) environment variable to `false` before running `cockroach demo`. + +## Shutting down and restarting nodes + + You can shut down and restart individual nodes in a multi-node demo cluster with the `\demo` SQL shell command. + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +Command | Description +----------------|------------ +`\demo shutdown ` | Shuts down a node.

This command simulates stopping a node that can be restarted. It is similar to [`cockroach quit`](cockroach-quit.html). +`\demo restart ` | Restarts a node that has been shut down. +`\demo decommission ` | Decommissions a node.

This command simulates [decommissioning a node](remove-nodes.html). It is similar to [`cockroach quit --decommission`](cockroach-quit.html#general). +`\demo recommission ` | Recommissions a decommissioned node. + +For examples, see [Shut down and restart nodes](cockroach-demo.html#shut-down-and-restart-nodes). + +## Examples + +In these examples, we demonstrate how to start a shell with `cockroach demo`. For more SQL shell features, see the [`cockroach sql` examples](cockroach-sql.html#examples). + +### Start an interactive SQL shell + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE drivers ( + id UUID DEFAULT gen_random_uuid(), + city STRING NOT NULL, + name STRING, + dl STRING UNIQUE, + address STRING, + CONSTRAINT primary_key PRIMARY KEY (city ASC, id ASC) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO drivers (city, name) VALUES ('new york', 'Catherine Nelson'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM drivers; +~~~ + +~~~ + id | city | name | dl | address ++--------------------------------------+----------+------------------+------+---------+ + df3dc272-b572-4ca4-88c8-e9974dbd381a | new york | Catherine Nelson | NULL | NULL +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ + +### Load a sample dataset and start an interactive SQL shell + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo movr --nodes=3 --demo-locality=region=us-east1:region=us-central1:region=us-west1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES; +~~~ + +~~~ + table_name ++----------------------------+ + promo_codes + rides + user_promo_codes + users + vehicle_location_histories + vehicles +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users WHERE city = 'new york'; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 +(5 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \q +~~~ + +### Execute SQL from the command-line + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo \ +--execute="CREATE TABLE drivers ( + id UUID DEFAULT gen_random_uuid(), + city STRING NOT NULL, + name STRING, + dl STRING UNIQUE, + address STRING, + CONSTRAINT primary_key PRIMARY KEY (city ASC, id ASC) +);" \ +--execute="INSERT INTO drivers (city, name) VALUES ('new york', 'Catherine Nelson');" \ +--execute="SELECT * FROM drivers;" +~~~ + +~~~ +CREATE TABLE +INSERT 1 + id | city | name | dl | address ++--------------------------------------+----------+------------------+------+---------+ + df3dc272-b572-4ca4-88c8-e9974dbd381a | new york | Catherine Nelson | NULL | NULL +(1 row) +~~~ + +### Run `cockroach demo` with a workload + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo --nodes=3 --with-load +~~~ + +This command starts a demo cluster with the `movr` database preloaded and then inserts rows into each table in the `movr` database. You can monitor the workload progress on the [Admin UI](admin-ui-overview-dashboard.html#sql-queries). + + When running a multi-node demo cluster, load is balanced across all nodes. + +### Run `cockroach demo` with geo-partitioned replicas + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo --geo-partitioned-replicas +~~~ + +This command starts a 9-node demo cluster with the `movr` database preloaded, and [partitions](partitioning.html) and [zone constraints](configure-replication-zones.html) applied to the primary and secondary indexes. For more information, see the [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) topology pattern. + +### Shut down and restart nodes + +If you start a demo cluster with multiple nodes, you can use the [`\demo`](cockroach-demo.html#shutting-down-and-restarting-nodes) shell command to shut down and restart individual nodes in the demo cluster. + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +For example, if you start a demo cluster with the following command: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach demo --nodes=3 +~~~ + +You can shutdown the 3rd node and then restart it: + +{% include copy-clipboard.html %} +~~~ sql +> \demo shutdown 3 +~~~ + +~~~ +node 3 has been shutdown +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \demo restart 3 +~~~ + +~~~ +node 3 has been restarted +~~~ + +You can also decommission the 3rd node and then recommission it: + +{% include copy-clipboard.html %} +~~~ sql +> \demo decommission 3 +~~~ + +~~~ +node 3 has been decommissioned +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \demo recommission 3 +~~~ + +~~~ +node 3 has been recommissioned +~~~ + +### Try your own scenario + +In addition to using one of the [pre-loaded dataset](#datasets), you can create your own database (e.g., [`CREATE DATABASE ;`](create-database.html)), or use the empty `defaultdb` database (e.g., [`SET DATABASE defaultdb;`](set-database.html)) to test our your own scenario involving any CockroachDB SQL features you are interested in. + +## See also + +- [`cockroach sql`](cockroach-sql.html) +- [`cockroach workload`](cockroach-workload.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [SQL Statements](sql-statements.html) +- [Learn CockroachDB SQL](learn-cockroachdb-sql.html) +- [MovR: Vehicle-Sharing App](movr.html) diff --git a/v20.2/cockroach-dump.md b/v20.2/cockroach-dump.md new file mode 100644 index 00000000000..829998b207f --- /dev/null +++ b/v20.2/cockroach-dump.md @@ -0,0 +1,409 @@ +--- +title: cockroach dump +summary: Learn how to dump schemas and data from a CockroachDB cluster. +toc: true +redirect_from: sql-dump.html +key: sql-dump.html +--- + +The `cockroach dump` [command](cockroach-commands.html) outputs the SQL statements required to recreate tables, views, and sequences. This command can be used to back up or export each database in a cluster. The output should also be suitable for importing into other relational databases, with minimal adjustments. + +{{site.data.alerts.callout_success}} +CockroachDB [enterprise license](https://www.cockroachlabs.com/pricing/) users can also back up their cluster's data using [`BACKUP`](backup.html). +{{site.data.alerts.end}} + +## Considerations + +When `cockroach dump` is executed: + +- Table, sequence, and view schemas and table data are dumped as they appeared at the time that the command is started. Any changes after the command starts will not be included in the dump. +- Table and view schemas are dumped in the order in which they can successfully be recreated. This is true of sequences as well. +- If the dump takes longer than the [`ttlseconds`](configure-replication-zones.html) replication setting for the table (25 hours by default), the dump may fail. +- Reads, writes, and schema changes can happen while the dump is in progress, but will not affect the output of the dump. + +{{site.data.alerts.callout_info}} +The user must have the `SELECT` privilege on the target table(s). +{{site.data.alerts.end}} + +## Synopsis + +Dump the schemas and data of specific tables to stdout: + +~~~ shell +$ cockroach dump
+~~~ + +Dump just the data of specific tables to stdout: + +~~~ shell +$ cockroach dump
--dump-mode=data +~~~ + +Dump just the schemas of specific tables to stdout: + +~~~ shell +$ cockroach dump
--dump-mode=schema +~~~ + +Dump the schemas and data of all tables in a database to stdout: + +~~~ shell +$ cockroach dump +~~~ + +Dump just the schemas of all tables in a database to stdout: + +~~~ shell +$ cockroach dump --dump-mode=schema +~~~ + +Dump just the data of all tables in a database to stdout: + +~~~ shell +$ cockroach dump --dump-mode=data +~~~ + +Dump to a file: + +~~~ shell +$ cockroach dump
> dump-file.sql +~~~ + +View help: + +~~~ shell +$ cockroach dump --help +~~~ + +## Flags + +The `dump` command supports the following [general-use](#general) and [logging](#logging) flags. + +### General + +Flag | Description +-----|------------ +`--as-of` | Dump table schema and/or data as they appear at the specified [timestamp](timestamp.html). See this [example](#dump-table-data-as-of-a-specific-time) for a demonstration.

Note that historical data is available only within the garbage collection window, which is determined by the [`ttlseconds`](configure-replication-zones.html) replication setting for the table (25 hours by default). If this timestamp is earlier than that window, the dump will fail.

**Default:** Current time +`--dump-mode` | Whether to dump table and view schemas, table data, or both.

To dump just table and view schemas, set this to `schema`. To dump just table data, set this to `data`. To dump both table and view schemas and table data, leave this flag out or set it to `both`.

Table and view schemas are dumped in the order in which they can successfully be recreated. For example, if a database includes a table, a second table with a foreign key dependency on the first, and a view that depends on the second table, the dump will list the schema for the first table, then the schema for the second table, and then the schema for the view.

**Default:** `both` +`--echo-sql` | Reveal the SQL statements sent implicitly by the command-line utility. + +### Client connection + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +See [Client Connection Parameters](connection-parameters.html) for more details. + +{{site.data.alerts.callout_info}} +The user specified with `--user` must have the `SELECT` privilege on the target tables. +{{site.data.alerts.end}} + +### Logging + +By default, the `dump` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +{{site.data.alerts.callout_info}} +These examples use our sample `startrek` database, which you can add to a cluster via the [`cockroach gen`](cockroach-gen.html#generate-example-data) command. Also, the examples assume that the `maxroach` user has been [granted](grant.html) the `SELECT` privilege on all target tables. +{{site.data.alerts.end}} + +### Dump a table's schema and data + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump startrek episodes --insecure --user=maxroach > backup.sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat backup.sql +~~~ + +~~~ +CREATE TABLE episodes ( + id INT NOT NULL, + season INT NULL, + num INT NULL, + title STRING NULL, + stardate DECIMAL NULL, + CONSTRAINT "primary" PRIMARY KEY (id), + FAMILY "primary" (id, season, num), + FAMILY fam_1_title (title), + FAMILY fam_2_stardate (stardate) +); + +INSERT INTO episodes (id, season, num, title, stardate) VALUES + (1, 1, 1, 'The Man Trap', 1531.1), + (2, 1, 2, 'Charlie X', 1533.6), + (3, 1, 3, 'Where No Man Has Gone Before', 1312.4), + (4, 1, 4, 'The Naked Time', 1704.2), + (5, 1, 5, 'The Enemy Within', 1672.1), + (6, 1, 6, e'Mudd\'s Women', 1329.8), + (7, 1, 7, 'What Are Little Girls Made Of?', 2712.4), + (8, 1, 8, 'Miri', 2713.5), + (9, 1, 9, 'Dagger of the Mind', 2715.1), + (10, 1, 10, 'The Corbomite Maneuver', 1512.2), + ... +~~~ + +### Dump just a table's schema + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump startrek episodes --insecure --user=maxroach --dump-mode=schema > backup.sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat backup.sql +~~~ + +~~~ +CREATE TABLE episodes ( + id INT NOT NULL, + season INT NULL, + num INT NULL, + title STRING NULL, + stardate DECIMAL NULL, + CONSTRAINT "primary" PRIMARY KEY (id), + FAMILY "primary" (id, season, num), + FAMILY fam_1_title (title), + FAMILY fam_2_stardate (stardate) +); +~~~ + +### Dump just a table's data + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump startrek episodes --insecure --user=maxroach --dump-mode=data > backup.sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat backup.sql +~~~ + +~~~ +INSERT INTO episodes (id, season, num, title, stardate) VALUES + (1, 1, 1, 'The Man Trap', 1531.1), + (2, 1, 2, 'Charlie X', 1533.6), + (3, 1, 3, 'Where No Man Has Gone Before', 1312.4), + (4, 1, 4, 'The Naked Time', 1704.2), + (5, 1, 5, 'The Enemy Within', 1672.1), + (6, 1, 6, e'Mudd\'s Women', 1329.8), + (7, 1, 7, 'What Are Little Girls Made Of?', 2712.4), + (8, 1, 8, 'Miri', 2713.5), + (9, 1, 9, 'Dagger of the Mind', 2715.1), + (10, 1, 10, 'The Corbomite Maneuver', 1512.2), + ... +~~~ + +### Dump all tables in a database + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump startrek --insecure --user=maxroach > backup.sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat backup.sql +~~~ + +~~~ +CREATE TABLE episodes ( + id INT NOT NULL, + season INT NULL, + num INT NULL, + title STRING NULL, + stardate DECIMAL NULL, + CONSTRAINT "primary" PRIMARY KEY (id), + FAMILY "primary" (id, season, num), + FAMILY fam_1_title (title), + FAMILY fam_2_stardate (stardate) +); + +CREATE TABLE quotes ( + quote STRING NULL, + characters STRING NULL, + stardate DECIMAL NULL, + episode INT NULL, + INDEX quotes_episode_idx (episode), + FAMILY "primary" (quote, rowid), + FAMILY fam_1_characters (characters), + FAMILY fam_2_stardate (stardate), + FAMILY fam_3_episode (episode) +); + +INSERT INTO episodes (id, season, num, title, stardate) VALUES + (1, 1, 1, 'The Man Trap', 1531.1), + (2, 1, 2, 'Charlie X', 1533.6), + (3, 1, 3, 'Where No Man Has Gone Before', 1312.4), + (4, 1, 4, 'The Naked Time', 1704.2), + (5, 1, 5, 'The Enemy Within', 1672.1), + (6, 1, 6, e'Mudd\'s Women', 1329.8), + (7, 1, 7, 'What Are Little Girls Made Of?', 2712.4), + (8, 1, 8, 'Miri', 2713.5), + (9, 1, 9, 'Dagger of the Mind', 2715.1), + (10, 1, 10, 'The Corbomite Maneuver', 1512.2), + ... + +INSERT INTO quotes (quote, characters, stardate, episode) VALUES + ('"... freedom ... is a worship word..." "It is our worship word too."', 'Cloud William and Kirk', NULL, 52), + ('"Beauty is transitory." "Beauty survives."', 'Spock and Kirk', NULL, 72), + ('"Can you imagine how life could be improved if we could do away with jealousy, greed, hate ..." "It can also be improved by eliminating love, tenderness, sentiment -- the other side of the coin"', 'Dr. Roger Corby and Kirk', 2712.4, 7), + ... +~~~ + +### Dump fails (user does not have `SELECT` privilege) + +In this example, the `dump` command fails for a user that does not have the `SELECT` privilege on the `episodes` table. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump startrek episodes --insecure --user=leslieroach > backup.sql +~~~ + +~~~ +Error: pq: user leslieroach has no privileges on table episodes +Failed running "dump" +~~~ + +### Restore a table from a backup file + +In this example, a user that has the `CREATE` privilege on the `startrek` database uses the [`cockroach sql`](cockroach-sql.html) command to recreate a table, based on a file created by the `dump` command. + +{% include copy-clipboard.html %} +~~~ shell +$ cat backup.sql +~~~ + +~~~ +CREATE TABLE quotes ( + quote STRING NULL, + characters STRING NULL, + stardate DECIMAL NULL, + episode INT NULL, + INDEX quotes_episode_idx (episode), + FAMILY "primary" (quote, rowid), + FAMILY fam_1_characters (characters), + FAMILY fam_2_stardate (stardate), + FAMILY fam_3_episode (episode) +); + +INSERT INTO quotes (quote, characters, stardate, episode) VALUES + ('"... freedom ... is a worship word..." "It is our worship word too."', 'Cloud William and Kirk', NULL, 52), + ('"Beauty is transitory." "Beauty survives."', 'Spock and Kirk', NULL, 72), + ('"Can you imagine how life could be improved if we could do away with jealousy, greed, hate ..." "It can also be improved by eliminating love, tenderness, sentiment -- the other side of the coin"', 'Dr. Roger Corby and Kirk', 2712.4, 7), + ... +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --database=startrek --user=maxroach < backup.sql +~~~ + +~~~ +CREATE TABLE +INSERT 100 +INSERT 100 +~~~ + +### Dump table data as of a specific time + +In this example, we assume there were several inserts into a table both before and after `2017-03-07 19:55:00`. + +First, let's use the built-in SQL client to view the table at the current time: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --execute="SELECT * FROM db1.dump_test" +~~~ + +~~~ ++--------------------+------+ +| id | name | ++--------------------+------+ +| 225594758537183233 | a | +| 225594758537248769 | b | +| 225594758537281537 | c | +| 225594758537314305 | d | +| 225594758537347073 | e | +| 225594758537379841 | f | +| 225594758537412609 | g | +| 225594758537445377 | h | +| 225594991654174721 | i | +| 225594991654240257 | j | +| 225594991654273025 | k | +| 225594991654305793 | l | +| 225594991654338561 | m | +| 225594991654371329 | n | +| 225594991654404097 | o | +| 225594991654436865 | p | ++--------------------+------+ +(16 rows) +~~~ + +Next, let's use a [time-travel query](select-clause.html#select-historical-data-time-travel) to view the contents of the table as of `2017-03-07 19:55:00`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --execute="SELECT * FROM db1.dump_test AS OF SYSTEM TIME '2017-03-07 19:55:00'" +~~~ + +~~~ ++--------------------+------+ +| id | name | ++--------------------+------+ +| 225594758537183233 | a | +| 225594758537248769 | b | +| 225594758537281537 | c | +| 225594758537314305 | d | +| 225594758537347073 | e | +| 225594758537379841 | f | +| 225594758537412609 | g | +| 225594758537445377 | h | ++--------------------+------+ +(8 rows) +~~~ + +Finally, let's use `cockroach dump` with the `--as-of` flag set to dump the contents of the table as of `2017-03-07 19:55:00`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach dump db1 dump_test --insecure --dump-mode=data --as-of='2017-03-07 19:55:00' +~~~ + +~~~ +INSERT INTO dump_test (id, name) VALUES + (225594758537183233, 'a'), + (225594758537248769, 'b'), + (225594758537281537, 'c'), + (225594758537314305, 'd'), + (225594758537347073, 'e'), + (225594758537379841, 'f'), + (225594758537412609, 'g'), + (225594758537445377, 'h'); +~~~ + +As you can see, the results of the dump are identical to the earlier time-travel query. + +## Known limitations + +### Dumping a table with no user-visible columns + +{% include {{page.version.version}}/known-limitations/dump-table-with-no-columns.md %} + +### Dumping a table with collations + +{% include {{page.version.version}}/known-limitations/dump-table-with-collations.md %} + +## See also + +- [Import Data](import-data.html) +- [`IMPORT`](import.html) +- [Use the Built-in SQL Client](cockroach-sql.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-gen.md b/v20.2/cockroach-gen.md new file mode 100644 index 00000000000..fbd55062212 --- /dev/null +++ b/v20.2/cockroach-gen.md @@ -0,0 +1,382 @@ +--- +title: cockroach gen +summary: Use cockroach gen to generate command-line interface utlities, such as man pages, and example data. +toc: true +redirect_from: generate-cockroachdb-resources.html +key: generate-cockroachdb-resources.html +--- + +The `cockroach gen` [command](cockroach-commands.html) can generate command-line interface (CLI) utilities ([`man` pages](https://en.wikipedia.org/wiki/Man_page) and a `bash` autocompletion script), example SQL data suitable to populate test databases, and an HAProxy configuration file for load balancing a running cluster. + +## Subcommands + +Subcommand | Usage +-----------|------ +`man` | Generate man pages for CockroachDB. +`autocomplete` | Generate `bash` or `zsh` autocompletion script for CockroachDB.

**Default:** `bash` +`example-data` | Generate example SQL datasets. You can also use the [`cockroach workload`](cockroach-workload.html) command to generate these sample datasets in a persistent cluster and the [`cockroach demo `](cockroach-demo.html) command to generate these datasets in a temporary, in-memory cluster. +`haproxy` | Generate an HAProxy config file for a running CockroachDB cluster. The node addresses included in the config are those advertised by the nodes. Make sure hostnames are resolvable and IP addresses are routable from HAProxy.

[Decommissioned nodes](remove-nodes.html) are excluded from the config file. + +## Synopsis + +Generate man pages: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen man +~~~ + +Generate bash autocompletion script: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen autocomplete +~~~ + +Generate example SQL data: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen example-data intro | cockroach sql +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen example-data startrek | cockroach sql +~~~ + +Generate an HAProxy config file for a running cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy +~~~ + +View help: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen --help +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen man --help +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen autocomplete --help +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen example-data --help +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy --help +~~~ + +## Flags + +The `gen` subcommands supports the following [general-use](#general) and [logging](#logging) flags. + +### General + +#### `man` + +Flag | Description +-----|----------- +`--path` | The path where man pages will be generated.

**Default:** `man/man1` under the current directory + +#### `autocomplete` + +Flag | Description +-----|----------- +`--out` | The path where the autocomplete file will be generated.

**Default:** `cockroach.bash` in the current directory + +#### `example-data` + +No flags are supported. See the [Generate Example Data](#generate-example-data) example for guidance. + +#### `haproxy` + +Flag | Description +-----|------------ +`--host` | The server host and port number to connect to. This can be the address of any node in the cluster.

**Env Variable:** `COCKROACH_HOST`
**Default:** `localhost:26257` +`--port`
`-p` | The server port to connect to. Note: The port number can also be specified via `--host`.

**Env Variable:** `COCKROACH_PORT`
**Default:** `26257` +`--insecure` | Use an insecure connection.

**Env Variable:** `COCKROACH_INSECURE`
**Default:** `false` +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html) containing the CA and client certificates and client key.

**Env Variable:** `COCKROACH_CERTS_DIR`
**Default:** `${HOME}/.cockroach-certs/` +`--url` | A [connection URL](connection-parameters.html#connect-using-a-url) to use instead of the other arguments.

**Env Variable:** `COCKROACH_URL`
**Default:** no URL +`--out` | The path where the `haproxy.cfg` file will be generated. If an `haproxy.cfg` file already exists in the directory, it will be overwritten.

**Default:** `haproxy.cfg` in the current directory +`--locality` | If nodes were started with [locality](cockroach-start.html#locality) details, you can use the `--locality` flag here to filter the nodes included in the HAProxy config file, specifying the explicit locality tier(s) or a regular expression to match against. This is useful in cases where you want specific instances of HAProxy to route to specific nodes. See the [Generate an HAProxy configuration file](#generate-an-haproxy-config-file) example for more details. + +### Logging + +By default, the `gen` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +### Generate `man` pages + +Generate man pages: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen man +~~~ + +Move the man pages to the man directory: + +{% include copy-clipboard.html %} +~~~ shell +$ sudo mv man/man1/* /usr/share/man/man1 +~~~ + +Access man pages: + +{% include copy-clipboard.html %} +~~~ shell +$ man cockroach +~~~ + +### Generate a `bash` autocompletion script + +Generate bash autocompletion script: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen autocomplete +~~~ + +Add the script to your `.bashrc` and `.bash_profle`: + +{% include copy-clipboard.html %} +~~~ shell +$ printf "\n\n#cockroach bash autocomplete\nsource 'cockroach.bash'" >> ~/.bashrc +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ printf "\n\n#cockroach bash autocomplete\nsource 'cockroach.bash'" >> ~/.bash_profile +~~~ + +You can now use `tab` to autocomplete `cockroach` commands. + +### Generate example data + +{{site.data.alerts.callout_success}} +You can also use the [`cockroach workload`](cockroach-workload.html) command to generate these sample datasets in a persistent cluster and the [`cockroach demo `](cockroach-demo.html) command to generate these datasets in a temporary, in-memory cluster. +{{site.data.alerts.end}} + +To test out CockroachDB, you can generate an example `startrek` database, which contains 2 tables, `episodes` and `quotes`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen example-data startrek | cockroach sql --insecure +~~~ + +~~~ +CREATE DATABASE +SET +DROP TABLE +DROP TABLE +CREATE TABLE +INSERT 79 +CREATE TABLE +INSERT 200 +~~~ + +Launch the built-in SQL client to view it: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM startrek; +~~~ +~~~ ++------------+ +| table_name | ++------------+ +| episodes | +| quotes | ++------------+ +(2 rows) +~~~ + +You can also generate an example `intro` database, which contains 1 table, `mytable`, with a hidden message: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen example-data intro | cockroach sql --insecure +~~~ + +~~~ +CREATE DATABASE +SET +DROP TABLE +CREATE TABLE +INSERT 1 +INSERT 1 +INSERT 1 +INSERT 1 +... +~~~ + +{% include copy-clipboard.html %} +~~~ shell +# Launch the built-in SQL client to view it: +$ cockroach sql --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM intro; +~~~ + +~~~ ++-------------+ +| table_name | ++-------------+ +| mytable | ++-------------+ +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM intro.mytable WHERE (l % 2) = 0; +~~~ + +~~~ ++----+------------------------------------------------------+ +| l | v | ++----+------------------------------------------------------+ +| 0 | !__aaawwmqmqmwwwaas,,_ .__aaawwwmqmqmwwaaa,, | +| 2 | !"VT?!"""^~~^"""??T$Wmqaa,_auqmWBT?!"""^~~^^""??YV^ | +| 4 | ! "?##mW##?"- | +| 6 | ! C O N G R A T S _am#Z??A#ma, Y | +| 8 | ! _ummY" "9#ma, A | +| 10 | ! vm#Z( )Xmms Y | +| 12 | ! .j####mmm#####mm#m##6. | +| 14 | ! W O W ! jmm###mm######m#mmm##6 | +| 16 | ! ]#me*Xm#m#mm##m#m##SX##c | +| 18 | ! dm#||+*$##m#mm#m#Svvn##m | +| 20 | ! :mmE=|+||S##m##m#1nvnnX##; A | +| 22 | ! :m#h+|+++=Xmm#m#1nvnnvdmm; M | +| 24 | ! Y $#m>+|+|||##m#1nvnnnnmm# A | +| 26 | ! O ]##z+|+|+|3#mEnnnnvnd##f Z | +| 28 | ! U D 4##c|+|+|]m#kvnvnno##P E | +| 30 | ! I 4#ma+|++]mmhvnnvq##P` ! | +| 32 | ! D I ?$#q%+|dmmmvnnm##! | +| 34 | ! T -4##wu#mm#pw##7' | +| 36 | ! -?$##m####Y' | +| 38 | ! !! "Y##Y"- | +| 40 | ! | ++----+------------------------------------------------------+ +(21 rows) +~~~ + +### Generate an HAProxy config file + +[HAProxy](http://www.haproxy.org/) is one of the most popular open-source TCP load balancers, and CockroachDB includes a built-in command for generating a configuration file that is preset to work with your running cluster. + +
+ + +

+ +
+To generate an HAProxy config file for an entire secure cluster, run the `cockroach gen haproxy` command, specifying the location of [certificate directory](cockroach-cert.html) and the address of any instance running a CockroachDB node: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy \ +--certs-dir= \ +--host=
+~~~ + +To limit the HAProxy config file to nodes matching specific ["localities"](cockroach-start.html#locality), use the `--localities` flag, specifying the explicit locality tier(s) or a regular expression to match against: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy \ +--certs-dir= \ +--host=
+--locality=region=us.* +~~~ +
+ +
+To generate an HAProxy config file for an entire insecure cluster, run the `cockroach gen haproxy` command, specifying the address of any instance running a CockroachDB node: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy \ +--insecure \ +--host=
+~~~ + +To limit the HAProxy config file to nodes matching specific ["localities"](cockroach-start.html#locality), use the `--localities` flag, specifying the explicit locality tier(s) or a regular expression to match against: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy \ +--insecure \ +--host=
+--locality=region=us.* +~~~ +
+ +By default, the generated configuration file is called `haproxy.cfg` and looks as follows, with the `server` addresses pre-populated correctly: + +~~~ +global + maxconn 4096 + +defaults + mode tcp + # Timeout values should be configured for your specific use. + # See: https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20connect + timeout connect 10s + timeout client 1m + timeout server 1m + # TCP keep-alive on client side. Server already enables them. + option clitcpka + +listen psql + bind :26257 + mode tcp + balance roundrobin + option httpchk GET /health?ready=1 + server cockroach1 :26257 check port 8080 + server cockroach2 :26257 check port 8080 + server cockroach3 :26257 check port 8080 +~~~ + +The file is preset with the minimal [configurations](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html) needed to work with your running cluster: + +Field | Description +------|------------ +`timeout connect`
`timeout client`
`timeout server` | Timeout values that should be suitable for most deployments. +`bind` | The port that HAProxy listens on. This is the port clients will connect to and thus needs to be allowed by your network configuration.

This tutorial assumes HAProxy is running on a separate machine from CockroachDB nodes. If you run HAProxy on the same machine as a node (not recommended), you'll need to change this port, as `26257` is likely already being used by the CockroachDB node. +`balance` | The balancing algorithm. This is set to `roundrobin` to ensure that connections get rotated amongst nodes (connection 1 on node 1, connection 2 on node 2, etc.). Check the [HAProxy Configuration Manual](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html#4-balance) for details about this and other balancing algorithms. +`option httpchk` | The HTTP endpoint that HAProxy uses to check node health. [`/health?ready=1`](monitoring-and-alerting.html#health-ready-1) ensures that HAProxy doesn't direct traffic to nodes that are live but not ready to receive requests. +`server` | For each included node, this field specifies the address the node advertises to other nodes in the cluster, i.e., the addressed pass in the [`--advertise-addr` flag](cockroach-start.html#networking) on node startup. Make sure hostnames are resolvable and IP addresses are routable from HAProxy. + +{{site.data.alerts.callout_info}} +For full details on these and other configuration settings, see the [HAProxy Configuration Manual](http://cbonte.github.io/haproxy-dconv/1.7/configuration.html). +{{site.data.alerts.end}} + +## See also + +- [Other Cockroach Commands](cockroach-commands.html) +- [Deploy CockroachDB On-Premises](deploy-cockroachdb-on-premises.html) (using HAProxy for load balancing) diff --git a/v20.2/cockroach-init.md b/v20.2/cockroach-init.md new file mode 100644 index 00000000000..2a7abf13c05 --- /dev/null +++ b/v20.2/cockroach-init.md @@ -0,0 +1,128 @@ +--- +title: cockroach init +summary: Perform a one-time-only initialization of a CockroachDB cluster. +toc: true +redirect_from: initialize-a-cluster.html +key: initialize-a-cluster.html +--- + +This page explains the `cockroach init` [command](cockroach-commands.html), which you use to perform a one-time initialization of a new multi-node cluster. For a full walk-through of the cluster startup and initialization process, see one of the [Manual Deployment](manual-deployment.html) tutorials. + +{{site.data.alerts.callout_info}} +When starting a single-node cluster with [`cockroach start-single-node`](cockroach-start-single-node.html), you do not need to use the `cockroach init` command. +{{site.data.alerts.end}} + +## Synopsis + +Perform a one-time initialization of a cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init +~~~ + +View help: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --help +~~~ + +## Flags + +The `cockroach init` command supports the following [client connection](#client-connection) and [logging](#logging) flags. + +### Client connection + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +See [Client Connection Parameters](connection-parameters.html) for details. + +### Logging + +By default, the `init` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +These examples assume that nodes have already been started with [`cockroach start`](cockroach-start.html) but are waiting to be initialized as a new cluster. For a more detailed walk-through, see one of the [Manual Deployment](manual-deployment.html) tutorials. + +### Initialize a Cluster on a Node's Machine + +
+ + +
+ +
+1. SSH to the machine where the node has been started. + +2. Make sure the `client.root.crt` and `client.root.key` files for the `root` user are on the machine. + +3. Run the `cockroach init` command with the `--certs-dir` flag set to the directory containing the `ca.crt` file and the files for the `root` user, and with the `--host` flag set to the address of the current node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init --certs-dir=certs --host=
+ ~~~ + + At this point, all the nodes complete startup and print helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the Admin UI, and the SQL URL for clients. +
+ +
+1. SSH to the machine where the node has been started. + +2. Run the `cockroach init` command with the `--host` flag set to the address of the current node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init --insecure --host=
+ ~~~ + + At this point, all the nodes complete startup and print helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the Admin UI, and the SQL URL for clients. +
+ +### Initialize a cluster from another machine + +
+ + +
+ +
+1. [Install the `cockroach` binary](install-cockroachdb.html) on a machine separate from the node. + +2. Create a `certs` directory and copy the CA certificate and the client certificate and key for the `root` user into the directory. + +3. Run the `cockroach init` command with the `--certs-dir` flag set to the directory containing the `ca.crt` file and the files for the `root` user, and with the `--host` flag set to the address of any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init --certs-dir=certs --host=
+ ~~~ + + At this point, all the nodes complete startup and print helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the Admin UI, and the SQL URL for clients. +
+ +
+1. [Install the `cockroach` binary](install-cockroachdb.html) on a machine separate from the node. + +2. Run the `cockroach init` command with the `--host` flag set to the address of any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init --insecure --host=
+ ~~~ + + At this point, all the nodes complete startup and print helpful details to the [standard output](cockroach-start.html#standard-output), such as the CockroachDB version, the URL for the Admin UI, and the SQL URL for clients. +
+ +## See also + +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Local Deployment](start-a-local-cluster.html) +- [`cockroach start`](cockroach-start.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-node.md b/v20.2/cockroach-node.md new file mode 100644 index 00000000000..383ae4077de --- /dev/null +++ b/v20.2/cockroach-node.md @@ -0,0 +1,298 @@ +--- +title: cockroach node +summary: To view details for each node in the cluster, use the cockroach node command with the appropriate subcommands and flags. +toc: true +redirect_from: view-node-details.html +key: view-node-details.html +--- + +To view details for each node in the cluster, use the `cockroach node` [command](cockroach-commands.html) with the appropriate subcommands and flags. + +The `cockroach node` command is also used in the process of decommissioning nodes for permanent removal. See [Remove Nodes](remove-nodes.html) for more details. + +## Subcommands + +Subcommand | Usage +-----------|------ +`ls` | List the ID of each node in the cluster, excluding those that have been decommissioned and are offline. +`status` | View the status of one or all nodes, excluding nodes that have been decommissioned and taken offline. Depending on flags used, this can include details about range/replicas, disk usage, and decommissioning progress. +`decommission` | Decommission nodes for permanent removal. See [Remove Nodes](remove-nodes.html) for more details. +`recommission` | Recommission nodes that were accidentally decommissioned. See [Recommission Nodes](remove-nodes.html#recommission-nodes) for more details. + +## Synopsis + +List the IDs of active and inactive nodes: + +~~~ shell +$ cockroach node ls +~~~ + +Show status details for active and inactive nodes: + +~~~ shell +$ cockroach node status +~~~ + +Show status and range/replica details for active and inactive nodes: + +~~~ shell +$ cockroach node status --ranges +~~~ + +Show status and disk usage details for active and inactive nodes: + +~~~ shell +$ cockroach node status --stats +~~~ + +Show status and decommissioning details for active and inactive nodes: + +~~~ shell +$ cockroach node status --decommission +~~~ + +Show complete status details for active and inactive nodes: + +~~~ shell +$ cockroach node status --all +~~~ + +Show status details for a specific node: + +~~~ shell +$ cockroach node status +~~~ + +Decommission nodes: + +~~~ shell +$ cockroach node decommission +~~~ + +Recommission nodes: + +~~~ shell +$ cockroach node recommission +~~~ + +View help: + +~~~ shell +$ cockroach node --help +~~~ +~~~ shell +$ cockroach node --help +~~~ + +## Flags + +All `node` subcommands support the following [general-use](#general) and [logging](#logging) flags. + +### General + +Flag | Description +-----|------------ +`--format` | How to display table rows printed to the standard output. Possible values: `tsv`, `csv`, `table`, `records`, `sql`, `html`.

**Default:** `tsv` + +The `node ls` subcommand also supports the following general flags: + +Flag | Description +-----|------------ +`--timeout` | Set the duration of time that the subcommand is allowed to run before it returns an error and prints partial information. The timeout is specified with a suffix of `s` for seconds, `m` for minutes, and `h` for hours. If this flag is not set, the subcommand may hang. + +The `node status` subcommand also supports the following general flags: + +Flag | Description +-----|------------ +`--all` | Show all node details. +`--decommission` | Show node decommissioning details. +`--ranges` | Show node details for ranges and replicas. +`--stats` | Show node disk usage details. +`--timeout` | Set the duration of time that the subcommand is allowed to run before it returns an error and prints partial information. The timeout is specified with a suffix of `s` for seconds, `m` for minutes, and `h` for hours. If this flag is not set, the subcommand may hang. + +The `node decommission` subcommand also supports the following general flag: + +Flag | Description +-----|------------ +`--wait` | When to return to the client. Possible values: `all`, `none`.

If `all`, the command returns to the client only after all specified nodes are fully decommissioned. If any specified nodes are offline, the command will not return to the client until those nodes are back online.

If `none`, the command does not wait for decommissioning to finish; it returns to the client after starting the decommissioning process on all specified nodes that are online. Any specified nodes that are offline will automatically be marked as decommissioned; if they come back online, the cluster will recognize this status and will not rebalance data to the nodes.

**Default:** `all` + +### Client connection + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +See [Client Connection Parameters](connection-parameters.html) for more details. + +### Logging + +By default, the `node` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Response + +The `cockroach node` subcommands return the following fields for each node. + +### `node ls` + +Field | Description +------|------------ +`id` | The ID of the node. + +### `node status` + +Field | Description +------|------------ +`id` | The ID of the node.

**Required flag:** None +`address` | The address of the node.

**Required flag:** None +`build` | The version of CockroachDB running on the node. If the binary was built from source, this will be the SHA hash of the commit used.

**Required flag:** None +`locality` | The [locality](cockroach-start.html#locality) information specified for the node.

**Required flag:** None +`updated_at` | The date and time when the node last recorded the information displayed in this command's output. When healthy, a new status should be recorded every 10 seconds or so, but when unhealthy this command's stats may be much older.

**Required flag:** None +`started_at` | The date and time when the node was started.

**Required flag:** None +`replicas_leaders` | The number of range replicas on the node that are the Raft leader for their range. See `replicas_leaseholders` below for more details.

**Required flag:** `--ranges` or `--all` +`replicas_leaseholders` | The number of range replicas on the node that are the leaseholder for their range. A "leaseholder" replica handles all read requests for a range and directs write requests to the range's Raft leader (usually the same replica as the leaseholder).

**Required flag:** `--ranges` or `--all` +`ranges` | The number of ranges that have replicas on the node.

**Required flag:** `--ranges` or `--all` +`ranges_unavailable` | The number of unavailable ranges that have replicas on the node.

**Required flag:** `--ranges` or `--all` +`ranges_underreplicated` | The number of underreplicated ranges that have replicas on the node.

**Required flag:** `--ranges` or `--all` +`live_bytes` | The amount of live data used by both applications and the CockroachDB system. This excludes historical and deleted data.

**Required flag:** `--stats` or `--all` +`key_bytes` | The amount of live and non-live data from keys in the key-value storage layer. This does not include data used by the CockroachDB system.

**Required flag:** `--stats` or `--all` +`value_bytes` | The amount of live and non-live data from values in the key-value storage layer. This does not include data used by the CockroachDB system.

**Required flag:** `--stats` or `--all` +`intent_bytes` | The amount of non-live data associated with uncommitted (or recently-committed) transactions.

**Required flag:** `--stats` or `--all` +`system_bytes` | The amount of data used just by the CockroachDB system.

**Required flag:** `--stats` or `--all` +`is_available` | If `true`, the node is currently available.

**Required flag:** None +`is_live` | If `true`, the node is currently live.

For unavailable clusters (with an unresponsive Admin UI), running the `node status` command and monitoring the `is_live` field is the only way to identify the live nodes in the cluster. However, you need to run the `node status` command on a live node to identify the other live nodes in an unavailable cluster. Figuring out a live node to run the command is a trial-and-error process, so run the command against each node until you get one that responds.

See [Identify live nodes in an unavailable cluster](#identify-live-nodes-in-an-unavailable-cluster) for more details.

**Required flag:** None +`gossiped_replicas` | The number of replicas on the node that are active members of a range. After decommissioning, this should be 0.

**Required flag:** `--decommission` or `--all` +`is_decommissioning` | If `true`, the node is marked for [decommissioning](remove-nodes.html).

**Required flag:** `--decommission` or `--all` +`is_draining` | If `true`, the range replicas and range leases are being moved off the node. This happens when a live node is being [decommissioned](remove-nodes.html).

**Required flag:** `--decommission` or `--all` + +### `node decommission` + +Field | Description +------|------------ +`id` | The ID of the node. +`is_live` | If `true`, the node is live. +`replicas` | The number of replicas on the node that are active members of a range. After decommissioning, this should be 0. +`is_decommissioning` | If `true`, the node is marked for [decommissioning](remove-nodes.html) +`is_draining` | If `true`, the range replicas and range leases are being moved off the node. This happens when a live node is being [decommissioned](remove-nodes.html). + +### `node recommission` + +Field | Description +------|------------ +`id` | The ID of the node. +`is_live` | If `true`, the node is live. +`replicas` | The number of replicas on the node that are active members of a range. After decommissioning, this should be 0. +`is_decommissioning` | If `true`, the node is marked for [decommissioning](remove-nodes.html). +`is_draining` | If `true`, the range replicas and range leases are being moved off the node. This happens when a live node is being [decommissioned](remove-nodes.html). + +## Examples + +### Setup + +To follow along with the examples, start [an insecure cluster](start-a-local-cluster.html), with [localities](cockroach-start.html#locality) defined. + +### List node IDs + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach node ls --insecure +~~~ + +~~~ + id ++----+ + 1 + 2 + 3 +(3 rows) +~~~ + +### Show the status of a single node + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach node status 1 --host=localhost:26257 --insecure +~~~ + +~~~ + id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live ++----+-----------------+-----------------+-----------------------------------------+----------------------------------+---------------------------------+---------------------+--------------+---------+ + 1 | localhost:26257 | localhost:26257 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.308502+00:00 | 2019-10-01 20:05:43.85563+00:00 | region=us-east,az=1 | true | true +(1 row) +~~~ + +### Show the status of all nodes + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach node status --host=localhost:26257 --insecure +~~~ + +~~~ + id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live ++----+-----------------+-----------------+-----------------------------------------+----------------------------------+----------------------------------+------------------------+--------------+---------+ + 1 | localhost:26257 | localhost:26257 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.308502+00:00 | 2019-10-01 20:06:15.356886+00:00 | region=us-east,az=1 | true | true + 2 | localhost:26258 | localhost:26258 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.551761+00:00 | 2019-10-01 20:06:15.583967+00:00 | region=us-central,az=2 | true | true + 3 | localhost:26259 | localhost:26259 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:55.178577+00:00 | 2019-10-01 20:06:16.204549+00:00 | region=us-west,az=3 | true | true +(3 rows) +~~~ + +### Identify live nodes in an unavailable cluster + +The `is_live` and `is_available` fields are marked as `true` as long as a majority of the nodes are up, and a quorum can be reached: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --host=localhost:26258 --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach node status --host=localhost:26257 --insecure +~~~ + +~~~ + id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live ++----+-----------------+-----------------+-----------------------------------------+----------------------------------+----------------------------------+------------------------+--------------+---------+ + 1 | localhost:26257 | localhost:26257 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.308502+00:00 | 2019-10-01 20:07:04.857339+00:00 | region=us-east,az=1 | true | true + 2 | localhost:26258 | localhost:26258 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.551761+00:00 | 2019-10-01 20:06:48.555863+00:00 | region=us-central,az=2 | false | false + 3 | localhost:26259 | localhost:26259 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:55.178577+00:00 | 2019-10-01 20:07:01.207697+00:00 | region=us-west,az=3 | true | true +(3 rows) +~~~ + +If a majority of nodes are down and a quorum cannot be reached, the `is_live` field is marked as `true` for the nodes that are up, but the `is_available` field is marked as `false` for all nodes: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --host=localhost:26259 --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach node status --host=localhost:26257 --insecure +~~~ + +~~~ + id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live ++----+-----------------+-----------------+-----------------------------------------+----------------------------------+----------------------------------+------------------------+--------------+---------+ + 1 | localhost:26257 | localhost:26257 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.308502+00:00 | 2019-10-01 20:07:37.464249+00:00 | region=us-east,az=1 | false | true + 2 | localhost:26258 | localhost:26258 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:54.551761+00:00 | 2019-10-01 20:07:37.464259+00:00 | region=us-central,az=2 | false | false + 3 | localhost:26259 | localhost:26259 | v19.2.0-alpha.20190606-2479-gd98e0839dc | 2019-10-01 20:04:55.178577+00:00 | 2019-10-01 20:07:37.464265+00:00 | region=us-west,az=3 | false | false +(3 rows) +~~~ + +{{site.data.alerts.callout_info}} +You need to run the `node status` command on a live node to identify the other live nodes in an unavailable cluster. Figuring out a live node to run the command is a trial-and-error process, so run the command against each node until you get one that responds. +{{site.data.alerts.end}} + +### Decommission nodes + +See [Remove Nodes](remove-nodes.html) + +### Recommission nodes + +See [Recommission Nodes](remove-nodes.html#recommission-nodes) + +## See also + +- [Other Cockroach Commands](cockroach-commands.html) +- [Remove Nodes](remove-nodes.html) diff --git a/v20.2/cockroach-nodelocal-upload.md b/v20.2/cockroach-nodelocal-upload.md new file mode 100644 index 00000000000..483cc298ac5 --- /dev/null +++ b/v20.2/cockroach-nodelocal-upload.md @@ -0,0 +1,97 @@ +--- +title: cockroach nodelocal upload +summary: Upload a file to a node's local file system. +toc: true +--- + + The `cockroach nodelocal upload` [command](cockroach-commands.html) uploads a file to the external IO directory on a node's (the gateway node, by default) local file system. + +This command takes in a source file to upload and a destination filename. It will then use a SQL connection to upload the file to the node's local file system, at `externalIODir/destination/filename`. + +{{site.data.alerts.callout_info}} +The source file is only uploaded to one node, not all of the nodes. +{{site.data.alerts.end}} + +## Required privileges + +Only members of the `admin` role can run `cockroach nodelocal upload`. By default, the `root` user belongs to the `admin` role. + +## Considerations + +The [`--external-io`](cockroach-start.html#general) flag on the node you're uploading to **cannot** be set to `disabled`. + +## Synopsis + +Upload a file: + +~~~ shell +$ cockroach nodelocal upload [flags] +~~~ + +View help: + +~~~ shell +$ cockroach nodelocal upload --help +~~~ + +## Flags + + Flag | Description +-----------------+----------------------------------------------------- +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html) containing the CA and client certificates and client key.

**Env Variable:** `COCKROACH_CERTS_DIR`
**Default:** `${HOME}/.cockroach-certs/` +`--echo-sql` | Reveal the SQL statements sent implicitly by the command-line utility. +`--host` | The server host and port number to connect to. This can be the address of any node in the cluster.

**Env Variable:** `COCKROACH_HOST`
**Default:** `localhost:26257` +`--insecure` | Use an insecure connection.

**Env Variable:** `COCKROACH_INSECURE`
**Default:** `false` +`--url` | A [connection URL](connection-parameters.html#connect-using-a-url) to use instead of the other arguments.

**Env Variable:** `COCKROACH_URL`
**Default:** no URL +`--user`
`-u` | The [SQL user](create-user.html) that will own the client session.

**Env Variable:** `COCKROACH_USER`
**Default:** `root` + +## Examples + +### Upload a file + +To upload a file to the default node (i.e., the gateway node): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach nodelocal upload ./grants.csv test/grants.csv --certs-dir=certs +~~~ + +~~~ +successfully uploaded to nodelocal://1/test/grants.csv +~~~ + +Then, you can use the file to [`IMPORT`](import.html) or [`IMPORT INTO`](import-into.html) data. + +### Upload a file to a specific node + +To upload a file to a specific node (e.g., node 2), use the `--host` flag: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach nodelocal upload ./grants.csv grants.csv --host=localhost:26259 --insecure +~~~ + +~~~ +successfully uploaded to nodelocal://2/grants.csv +~~~ + +Or, use the `--url` flag: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach nodelocal upload ./grants.csv grants.csv --url=postgresql://root@localhost:26258?sslmode=disable --insecure +~~~ + +~~~ +successfully uploaded to nodelocal://3/grants.csv +~~~ + +Then, you can use the file to [`IMPORT`](import.html) or [`IMPORT INTO`](import-into.html) data. + +## See also + +- [Other Cockroach Commands](cockroach-commands.html) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Import Data](import-data.html) +- [`IMPORT`](import.html) +- [`IMPORT INTO`](import-into.html) diff --git a/v20.2/cockroach-quit.md b/v20.2/cockroach-quit.md new file mode 100644 index 00000000000..f6f892ddc16 --- /dev/null +++ b/v20.2/cockroach-quit.md @@ -0,0 +1,135 @@ +--- +title: cockroach quit +summary: Learn how to temporarily stop a CockroachDB node. +toc: true +redirect_from: stop-a-node.html +key: stop-a-node.html +--- + +This page shows you how to use the `cockroach quit` [command](cockroach-commands.html) to temporarily stop a node that you plan to restart, for example, during the process of [upgrading your cluster's version of CockroachDB](upgrade-cockroach-version.html) or to perform planned maintenance (e.g., upgrading system software). + +For information about permanently removing nodes to downsize a cluster or react to hardware failures, see [Remove Nodes](remove-nodes.html). + +## Overview + +### How it works + +When you stop a node, it performs the following steps: + +- Finishes in-flight requests. Note that this is a best effort that times out after the duration specified by the `server.shutdown.query_wait` [cluster setting](cluster-settings.html). +- Transfers all **range leases** and Raft leadership to other nodes. +- Gossips its draining state to the cluster, so that other nodes do not try to distribute query planning to the draining node, and no leases are transferred to the draining node. Note that this is a best effort that times out after the duration specified by the `server.shutdown.drain_wait` [cluster setting](cluster-settings.html), so other nodes may not receive the gossip info in time. +- No new ranges are transferred to the draining node, to avoid a possible loss of quorum after the node shuts down. + +If the node then stays offline for a certain amount of time (5 minutes by default), the cluster considers the node dead and starts to transfer its **range replicas** to other nodes as well. + +After that, if the node comes back online, its range replicas will determine whether or not they are still valid members of replica groups. If a range replica is still valid and any data in its range has changed, it will receive updates from another replica in the group. If a range replica is no longer valid, it will be removed from the node. + +Basic terms: + +- **Range**: CockroachDB stores all user data and almost all system data in a giant sorted map of key value pairs. This keyspace is divided into "ranges", contiguous chunks of the keyspace, so that every key can always be found in a single range. +- **Range Replica:** CockroachDB replicates each range (3 times by default) and stores each replica on a different node. +- **Range Lease:** For each range, one of the replicas holds the "range lease". This replica, referred to as the "leaseholder", is the one that receives and coordinates all read and write requests for the range. + +### Considerations + +{% include {{ page.version.version }}/faq/planned-maintenance.md %} + +## Synopsis + +Temporarily stop a node: + +~~~ shell +$ cockroach quit +~~~ + +View help: + +~~~ shell +$ cockroach quit --help +~~~ + +## Flags + +The `quit` command supports the following [general-use](#general), [client connection](#client-connection), and [logging](#logging) flags. + +### General + +Flag | Description +-----|------------ +`--decommission` | If specified, the node will be permanently removed instead of temporarily stopped. See [Remove Nodes](remove-nodes.html) for more details. + +### Client connection + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +See [Client Connection Parameters](connection-parameters.html) for more details. + +### Logging + +By default, the `quit` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +### Stop a node from the machine where it's running + +1. SSH to the machine where the node is running. + +2. If the node is running in the background and you are using a process manager for automatic restarts, use the process manager to stop the `cockroach` process without restarting it. + + If the node is running in the background and you are not using a process manager, send a kill signal to the `cockroach` process, for example: + + {% include copy-clipboard.html %} + ~~~ shell + $ pkill cockroach + ~~~ + + If the node is running in the foreground, press `CTRL-C`. + +3. Verify that the `cockroach` process has stopped: + + {% include copy-clipboard.html %} + ~~~ shell + $ ps aux | grep cockroach + ~~~ + + Alternately, you can check the node's logs for the message `server drained and shutdown completed`. + +### Stop a node from another machine + +
+ + +
+ +
+1. [Install the `cockroach` binary](install-cockroachdb.html) on a machine separate from the node. + +2. Create a `certs` directory and copy the CA certificate and the client certificate and key for the `root` user into the directory. + +3. Run the `cockroach quit` command without the `--decommission` flag: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --certs-dir=certs --host=
+ ~~~ +
+ +
+1. [Install the `cockroach` binary](install-cockroachdb.html) on a machine separate from the node. + +2. Run the `cockroach quit` command without the `--decommission` flag: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=
+ ~~~ +
+ +## See also + +- [Other Cockroach Commands](cockroach-commands.html) +- [Permanently Remove Nodes from a Cluster](remove-nodes.html) +- [Upgrade a Cluster's Version](upgrade-cockroach-version.html) diff --git a/v20.2/cockroach-sql.md b/v20.2/cockroach-sql.md new file mode 100644 index 00000000000..d128910b1d1 --- /dev/null +++ b/v20.2/cockroach-sql.md @@ -0,0 +1,839 @@ +--- +title: cockroach sql +summary: CockroachDB comes with a built-in client for executing SQL statements from an interactive shell or directly from the command line. +toc: true +redirect_from: use-the-built-in-sql-client.html +key: use-the-built-in-sql-client.html +--- + +CockroachDB comes with a built-in client for executing SQL statements from an interactive shell or directly from the command line. To use this client, run the `cockroach sql` [command](cockroach-commands.html) as described below. + +To exit the interactive shell, use `\q`, `quit`, `exit`, or `ctrl-d`. + +{{site.data.alerts.callout_success}} +If you want to experiment with CockroachDB SQL but do not have a cluster already running, you can use the [`cockroach demo`](cockroach-demo.html) command to open a shell to a temporary, in-memory cluster. +{{site.data.alerts.end}} + +## Synopsis + +Start the interactive SQL shell: + +~~~ shell +$ cockroach sql +~~~ + +Execute SQL from the command line: + +~~~ shell +$ cockroach sql --execute=";" --execute="" +~~~ +~~~ shell +$ echo ";" | cockroach sql +~~~ +~~~ shell +$ cockroach sql < file-containing-statements.sql +~~~ + +Exit the interactive SQL shell: + +~~~ shell +$ \q +~~~ +~~~ shell +$ quit +~~~ +~~~ shell +$ exit +~~~ +~~~ shell +ctrl-d +~~~ + +View help: + +~~~ shell +$ cockroach sql --help +~~~ + +## Flags + +The `sql` command supports the following types of flags: + +- [General Use](#general) +- [Client Connection](#client-connection) +- [Logging](#logging) + +### General + +- To start an interactive SQL shell, run `cockroach sql` with all appropriate connection flags or use just the [`--url` flag](#sql-flag-url), which includes [connection details](connection-parameters.html#connect-using-a-url). +- To execute SQL statements from the command line, use the [`--execute` flag](#sql-flag-execute). + +Flag | Description +-----|------------ +`--database`
`-d` | A database name to use as [current database](sql-name-resolution.html#current-database) in the newly created session. +`--echo-sql` | Reveal the SQL statements sent implicitly by the command-line utility. For a demonstration, see the [example](#reveal-the-sql-statements-sent-implicitly-by-the-command-line-utility) below.

This can also be enabled within the interactive SQL shell via the `\set echo` [shell command](#commands). + `--execute`
`-e` | Execute SQL statements directly from the command line, without opening a shell. This flag can be set multiple times, and each instance can contain one or more statements separated by semi-colons. If an error occurs in any statement, the command exits with a non-zero status code and further statements are not executed. The results of each statement are printed to the standard output (see `--format` for formatting options).

For a demonstration of this and other ways to execute SQL from the command line, see the [example](#execute-sql-statements-from-the-command-line) below. + `--format` | How to display table rows printed to the standard output. Possible values: `tsv`, `csv`, `table`, `raw`, `records`, `sql`, `html`.

**Default:** `table` for sessions that [output on a terminal](#session-and-output-types); `tsv` otherwise

This flag corresponds to the `display_format` [client-side option](#client-side-options). +`--safe-updates` | Disallow potentially unsafe SQL statements, including `DELETE` without a `WHERE` clause, `UPDATE` without a `WHERE` clause, and `ALTER TABLE ... DROP COLUMN`.

**Default:** `true` for [interactive sessions](#session-and-output-types); `false` otherwise

Potentially unsafe SQL statements can also be allowed/disallowed for an entire session via the `sql_safe_updates` [session variable](set-vars.html). +`--set` | Set a [client-side option](#client-side-options) before starting the SQL shell or executing SQL statements from the command line via `--execute`. This flag may be specified multiple times, once per option.

After starting the SQL shell, the `\set` and `unset` commands can be use to enable and disable client-side options as well. +`--watch` | Repeat the SQL statements specified with `--execute` or `-e` until a SQL error occurs or the process is terminated. `--watch` applies to all `--execute` or `-e` flags in use.
You must also specify an interval at which to repeat the statement, followed by a time unit. For example, to specify an interval of 5 seconds, use `5s`.

Note that this flag is intended for simple monitoring scenarios during development and testing. See the [example](#repeat-a-sql-statement) below. + +### Client connection + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +See [Client Connection Parameters](connection-parameters.html) for more details. + +### Logging + +By default, the `sql` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Session and output types + +`cockroach sql` exhibits different behaviors depending on whether or not the session is interactive and/or whether or not the session outputs on a terminal. + +- A session is **interactive** when `cockroach sql` is invoked without the `--execute` flag and input is not redirected from a file. In such cases: + - The [`errexit` option](#sql-option-errexit) defaults to `false`. + - The [`check_syntax` option](#sql-option-check-syntax) defaults to `true` if supported by the CockroachDB server (this is checked when the shell starts up). + - **Ctrl+C** at the prompt will only terminate the shell if no other input was entered on the same line already. + - The shell will attempt to set the `safe_updates` [session variable](set-vars.html) to `true` on the server. +- A session **outputs on a terminal** when output is not redirected to a file. In such cases: + - The [`--format` flag](#sql-flag-format) and its corresponding [`display_format` option](#sql-option-display-format) default to `table`. These default to `tsv` otherwise. + - The `show_times` option defaults to `true`. + +When a session is both interactive and outputs on a terminal, `cockroach sql` also activates the interactive prompt with a line editor that can be used to modify the current line of input. Also, command history becomes active. + +## SQL shell + +### Welcome message + +When the SQL shell connects (or reconnects) to a CockroachDB node, it prints a welcome text with some tips and CockroachDB version and cluster details: + +~~~ shell +# +# Welcome to the CockroachDB SQL shell. +# All statements must be terminated by a semicolon. +# To exit, type: \q. +# +# Server version: CockroachDB CCL {{page.release_info.version}} (x86_64-apple-darwin17.7.0, built 2019/09/13 00:07:19, go1.12.6) (same version as client) +# Cluster ID: 7fb9f5b4-a801-4851-92e9-c0db292d03f1 +# +# Enter \? for a brief introduction. +# +> +~~~ + +The **Version** and **Cluster ID** details are particularly noteworthy: + +- When the client and server versions of CockroachDB are the same, the shell prints the `Server version` followed by `(same version as client)`. +- When the client and server versions are different, the shell prints both the `Client version` and `Server version`. In this case, you may want to [plan an upgrade](upgrade-cockroach-version.html) of older client or server versions. +- Since every CockroachDB cluster has a unique ID, you can use the `Cluster ID` field to verify that your client is always connecting to the correct cluster. + +### Commands + +The following commands can be used within the interactive SQL shell: + +Command | Usage +--------|------------ +`\?`
`help` | View this help within the shell. +`\q`
`quit`
`exit`
`ctrl-d` | Exit the shell.

When no text follows the prompt, `ctrl-c` exits the shell as well; otherwise, `ctrl-c` clears the line. +`\!` | Run an external command and print its results to `stdout`. See the [example](#run-external-commands-from-the-sql-shell) below. +\| | Run the output of an external command as SQL statements. See the [example](#run-external-commands-from-the-sql-shell) below. +`\set
` | Show details about columns in the specified table. This command is equivalent to [`SHOW COLUMNS`](show-columns.html). + +### Client-side options + +- To view option descriptions and how they are currently set, use `\set` without any options. +- To enable or disable an option, use `\set
+ + + + +
chickturtle
🐥🐢
+~~~ + +When piping output to another command or a file, `--format` defaults to `tsv`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--execute="SELECT '🐥' AS chick, '🐢' AS turtle" > out.txt \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=critterdb +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat out.txt +~~~ + +~~~ +1 row +chick turtle +🐥 🐢 +~~~ + +However, you can explicitly set `--format` to another format, for example, `table`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--format=table \ +--execute="SELECT '🐥' AS chick, '🐢' AS turtle" > out.txt \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=critterdb +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cat out.txt +~~~ + +~~~ ++-------+--------+ +| chick | turtle | ++-------+--------+ +| 🐥 | 🐢 | ++-------+--------+ +(1 row) +~~~ + +### Make the output of `SHOW` statements selectable + +To make it possible to select from the output of `SHOW` statements, set `--format` to `raw`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--format=raw \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=critterdb +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE customers; +~~~ + +~~~ +# 2 columns +# row 1 +## 14 +test.customers +## 185 +CREATE TABLE customers ( + id INT NOT NULL, + email STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (id ASC), + UNIQUE INDEX customers_email_key (email ASC), + FAMILY "primary" (id, email) +) +# 1 row +~~~ + +When `--format` is not set to `raw`, you can use the `display_format` [SQL shell option](#client-side-options) to change the output format within the interactive session: + +{% include copy-clipboard.html %} +~~~ sql +> \set display_format raw +~~~ + +~~~ +# 2 columns +# row 1 +## 14 +test.customers +## 185 +CREATE TABLE customers ( + id INT NOT NULL, + email STRING NULL, + CONSTRAINT "primary" PRIMARY KEY (id ASC), + UNIQUE INDEX customers_email_key (email ASC), + FAMILY "primary" (id, email) +) +# 1 row +~~~ + +### Execute SQL statements from a file + +In this example, we show and then execute the contents of a file containing SQL statements. + +{% include copy-clipboard.html %} +~~~ shell +$ cat statements.sql +~~~ + +~~~ +CREATE TABLE roaches (name STRING, country STRING); +INSERT INTO roaches VALUES ('American Cockroach', 'United States'), ('Brownbanded Cockroach', 'United States'); +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=critterdb \ +< statements.sql +~~~ + +~~~ +CREATE TABLE +INSERT 2 +~~~ + +### Run external commands from the SQL shell + +In this example, we use `\!` to look at the rows in a CSV file before creating a table and then using `\|` to insert those rows into the table. + +{{site.data.alerts.callout_info}}This example works only if the values in the CSV file are numbers. For values in other formats, use an online CSV-to-SQL converter or make your own import program.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> \! cat test.csv +~~~ + +~~~ +12, 13, 14 +10, 20, 30 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE csv (x INT, y INT, z INT); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \| IFS=","; while read a b c; do echo "insert into csv values ($a, $b, $c);"; done < test.csv; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM csv; +~~~ + +~~~ ++----+----+----+ +| x | y | z | ++----+----+----+ +| 12 | 13 | 14 | +| 10 | 20 | 30 | ++----+----+----+ +~~~ + +In this example, we create a table and then use `\|` to programmatically insert values. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE for_loop (x INT); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \| for ((i=0;i<10;++i)); do echo "INSERT INTO for_loop VALUES ($i);"; done +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM for_loop; +~~~ + +~~~ ++---+ +| x | ++---+ +| 0 | +| 1 | +| 2 | +| 3 | +| 4 | +| 5 | +| 6 | +| 7 | +| 8 | +| 9 | ++---+ +~~~ + +### Edit SQL statements in an external editor + +In applications that use [GNU Readline](https://tiswww.case.edu/php/chet/readline/rltop.html) (such as [bash](https://www.gnu.org/software/bash/)), you can edit a long line in your preferred editor by typing `Ctrl-x Ctrl-e`. However, CockroachDB uses the BSD-licensed [libedit](https://thrysoee.dk/editline/), which does not include this functionality. + +If you would like to be able to edit the current line in an external editor by typing `C-x C-e` as in `bash`, do the following: + +1. Install the `vipe` program (from the [moreutils](https://joeyh.name/code/moreutils/) suite of tools). +2. Edit your `~/.editrc` to add the following line, which takes advantage of the SQL client's ability to [run external commands](#run-external-commands-from-the-sql-shell): + + {% include copy-clipboard.html %} + ~~~ + cockroach:bind -s ^X^E '^A^K\\\| echo \"^Y\" | vipe\r' + ~~~ + +This tells libedit to translate `C-x C-e` into the following commands: + +1. Move to the beginning of the current line. +2. Cut the whole line. +3. Paste the line into your editor via `vipe`. +4. Pass the edited file back to the SQL client when `vipe` exits. + +{{site.data.alerts.callout_info}} +Future versions of the SQL client may opt to use a different back-end for reading input, in which case please refer to this page for additional updates. +{{site.data.alerts.end}} + +### Allow potentially unsafe SQL statements + +The `--safe-updates` flag defaults to `true`. This prevents SQL statements that may have broad, undesired side-effects. For example, by default, we cannot use `DELETE` without a `WHERE` clause to delete all rows from a table: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --execute="SELECT * FROM db1.t1" +~~~ + +~~~ ++----+------+ +| id | name | ++----+------+ +| 1 | a | +| 2 | b | +| 3 | c | +| 4 | d | +| 5 | e | +| 6 | f | +| 7 | g | +| 8 | h | +| 9 | i | +| 10 | j | ++----+------+ +(10 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --execute="DELETE FROM db1.t1" +~~~ + +~~~ +Error: pq: rejected: DELETE without WHERE clause (sql_safe_updates = true) +Failed running "sql" +~~~ + +However, to allow an "unsafe" statement, you can set `--safe-updates=false`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --safe-updates=false --execute="DELETE FROM db1.t1" +~~~ + +~~~ +DELETE 10 +~~~ + +{{site.data.alerts.callout_info}}Potentially unsafe SQL statements can also be allowed/disallowed for an entire session via the sql_safe_updates session variable.{{site.data.alerts.end}} + +### Reveal the SQL statements sent implicitly by the command-line utility + +In this example, we use the `--execute` flag to execute statements from the command line and the `--echo-sql` flag to reveal SQL statements sent implicitly: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--execute="CREATE TABLE t1 (id INT PRIMARY KEY, name STRING)" \ +--execute="INSERT INTO t1 VALUES (1, 'a'), (2, 'b'), (3, 'c')" \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=db1 +--echo-sql +~~~ + +~~~ +# Server version: CockroachDB CCL f8f3c9317 (darwin amd64, built 2017/09/13 15:05:35, go1.8) (same version as client) +# Cluster ID: 847a4ba5-c78a-465a-b1a0-59fae3aab520 +> SET sql_safe_updates = TRUE +> CREATE TABLE t1 (id INT PRIMARY KEY, name STRING) +CREATE TABLE +> INSERT INTO t1 VALUES (1, 'a'), (2, 'b'), (3, 'c') +INSERT 3 +~~~ + +In this example, we start the interactive SQL shell and enable the `echo` shell option to reveal SQL statements sent implicitly: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--user=maxroach \ +--host=12.345.67.89 \ +--database=db1 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> \set echo +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO db1.t1 VALUES (4, 'd'), (5, 'e'), (6, 'f'); +~~~ + +~~~ +> INSERT INTO db1.t1 VALUES (4, 'd'), (5, 'e'), (6, 'f'); +INSERT 3 + +Time: 2.426534ms + +> SHOW TRANSACTION STATUS +> SHOW DATABASE +~~~ + +### Repeat a SQL statement + +Repeating SQL queries on a table can be useful for monitoring purposes. With the `--watch` flag, you can repeat the statements specified with a `--execute` or `-e` flag periodically, until a SQL error occurs or the process is terminated. + +For example, if you want to monitor the number of queries running on the current node, you can use `cockroach sql` with the `--watch` flag to query the node's `crdb_internal.node_statement_statistics` table for the query count: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure \ +--execute="SELECT SUM(count) FROM crdb_internal.node_statement_statistics" \ +--watch 1m +~~~ + +~~~ + sum ++-----+ + 926 +(1 row) + sum ++------+ + 4227 +(1 row) +^C +~~~ + +In this example, the statement is executed every minute. We let the process run for a couple minutes before killing it with Ctrl+C. + +### Connect to a cluster listening for Unix domain socket connections + + To connect to a cluster that is running on the same machine as your client and is listening for [Unix domain socket](https://en.wikipedia.org/wiki/Unix_domain_socket) connections, [specify a Unix domain socket URI](connection-parameters.html#example-uri-for-a-unix-domain-socket) with the `--url` connection parameter. + +For example, suppose you start a single-node cluster with the following [`cockroach start-single-node`](cockroach-start-single-node.html) command: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start-single-node --insecure --socket-dir=/tmp +~~~ + +~~~ +CockroachDB node starting at 2020-04-22 15:07:17.232326 +0000 UTC (took 0.9s) +build: CCL v20.2.0 @ 2020/06/02 13:54:06 (go1.13.4) +webui: http://localhost:8080 +sql: postgresql://root@localhost:26257?sslmode=disable +RPC client flags: ./cockroach --host=localhost:26257 --insecure +socket: /tmp/.s.PGSQL.26257 +logs: /path/cockroach/cockroach-data/logs +temp dir: /path/cockroach/cockroach-data/cockroach-temp919020614 +external I/O path: /path/cockroach/cockroach-data/extern +store[0]: path=/path/cockroach/cockroach-data +storage engine: rocksdb +status: restarted pre-existing node +clusterID: 9ce204b4-4b79-4809-83b5-2dc54c190cb2 +nodeID: 1 +~~~ + +To connect to this cluster with a socket: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --url='postgres://@?host=/tmp&port=26257' +~~~ + +## See also + +- [Client Connection Parameters](connection-parameters.html) +- [`cockroach demo`](cockroach-demo.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [SQL Statements](sql-statements.html) +- [Learn CockroachDB SQL](learn-cockroachdb-sql.html) diff --git a/v20.2/cockroach-sqlfmt.md b/v20.2/cockroach-sqlfmt.md new file mode 100644 index 00000000000..08df7e94c2f --- /dev/null +++ b/v20.2/cockroach-sqlfmt.md @@ -0,0 +1,159 @@ +--- +title: cockroach sqlfmt +summary: Use cockroach sqlfmt to enhance the text layout of a SQL query. +toc: true +redirect_from: use-the-query-formatter.html +key: use-the-query-formatter.html +--- + +The `cockroach sqlfmt` +[command](cockroach-commands.html) changes the textual formatting of +one or more SQL queries. It recognizes all SQL extensions supported by +CockroachDB. + +A [web interface to this feature](https://sqlfum.pt/) is also available. + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +## Synopsis + +Use the query formatter interactively: + +~~~ shell +$ cockroach sqlfmt + +CTRL+D +~~~ + +Reformat a SQL query given on the command line: + +~~~ shell +$ cockroach sqlfmt -e "" +~~~ + +Reformat a SQL query already stored in a file: + +~~~ shell +$ cat query.sql | cockroach sqlfmt +~~~ + +## Flags + +The `sqlfmt` command supports the following flags. + +Flag | Description | Default value +-----|------|---- +`--execute`
`-e` | Reformat the given SQL query, without reading from standard input. | N/A +`--print-width` | Desired column width of the output. | 80 +`--tab-width` | Number of spaces occupied by a tab character on the final display device. | 4 +`--use-spaces` | Always use space characters for formatting; avoid tab characters. | Use tabs. +`--align` | Use vertical alignment during formatting. | Do not align vertically. +`--no-simplify` | Avoid removing optional grouping parentheses during formatting. | Remove unnecessary grouping parentheses. + +## Examples + +### Reformat a query with constrained column width + +Using the interactive query formatter, output with the default column width (80 columns): + +1. Start the interactive query formatter: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sqlfmt + ~~~ + +2. Press **Enter**. + +3. Run the query: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE animals (id INT PRIMARY KEY DEFAULT unique_rowid(), name STRING); + ~~~ +4. Press **CTRL+D**. + + ~~~ sql + CREATE TABLE animals ( + id INT PRIMARY KEY DEFAULT unique_rowid(), + name STRING + ) + ~~~ + +Using the command line, output with the column width set to `40`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sqlfmt --print-width 40 -e "CREATE TABLE animals (id INT PRIMARY KEY DEFAULT unique_rowid(), name STRING);" +~~~ + +~~~ sql +CREATE TABLE animals ( + id + INT + PRIMARY KEY + DEFAULT unique_rowid(), + name STRING +) +~~~ + +### Reformat a query with vertical alignment + +Output with the default vertical alignment: + +~~~ shell +$ cockroach sqlfmt -e "SELECT winner, round(length / (60 * 5)) AS counter FROM players WHERE build = $1 AND (hero = $2 OR region = $3);" +~~~ + +~~~ sql +SELECT +winner, round(length / (60 * 5)) AS counter +FROM +players +WHERE +build = $1 AND (hero = $2 OR region = $3) +~~~ + +Output with vertical alignment: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sqlfmt --align -e "SELECT winner, round(length / (60 * 5)) AS counter FROM players WHERE build = $1 AND (hero = $2 OR region = $3);" +~~~ + +~~~ sql +SELECT winner, round(length / (60 * 5)) AS counter + FROM players + WHERE build = $1 AND (hero = $2 OR region = $3); +~~~ + +### Reformat a query with simplification of parentheses + +Output with the default simplification of parentheses: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sqlfmt -e "SELECT (1 * 2) + 3, (1 + 2) * 3;" +~~~ + +~~~ sql +SELECT 1 * 2 + 3, (1 + 2) * 3 +~~~ + +Output with no simplification of parentheses: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sqlfmt --no-simplify -e "SELECT (1 * 2) + 3, (1 + 2) * 3;" +~~~ + +~~~ sql +SELECT (1 * 2) + 3, (1 + 2) * 3 +~~~ + +## See also + +- [Sequel Fumpt](https://sqlfum.pt/) +- [`cockroach demo`](cockroach-demo.html) +- [`cockroach sql`](cockroach-sql.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-start-single-node.md b/v20.2/cockroach-start-single-node.md new file mode 100644 index 00000000000..dfd459047a0 --- /dev/null +++ b/v20.2/cockroach-start-single-node.md @@ -0,0 +1,434 @@ +--- +title: cockroach start-single-node +summary: Start a single-node cluster for app development. +toc: true +--- + +This page explains the `cockroach start-single-node` [command](cockroach-commands.html), which you use to start a single-node cluster with replication disabled. A single-node cluster is all you need for quick SQL testing or app development. + +{{site.data.alerts.callout_success}} +To run a multi-node cluster with replicated data for availability and consistency, use [`cockroach start`](cockroach-start.html) and [`cockroach init`](cockroach-init.html). +{{site.data.alerts.end}} + +## Synopsis + +Start a single-node cluster: + +~~~ shell +$ cockroach start-single-node +~~~ + +View help: + +~~~ shell +$ cockroach start-single-node --help +~~~ + +## Flags + +The `cockroach start-single-node` command supports the following [general-use](#general), [networking](#networking), [security](#security), and [logging](#logging) flags. + +Many flags have useful defaults that can be overridden by specifying the flags explicitly. If you specify flags explicitly, however, be sure to do so each time the node is restarted, as they will not be remembered. + +{{site.data.alerts.callout_info}} +The `cockroach start-single-node` flags are identical to [`cockroach start`](cockroach-start.html#flags) flags. However, many of them are not relevant for single-node clusters but are provided for users who want to test concepts that appear in multi-node clusters. These flags are called out as such. In most cases, accepting most defaults is sufficient (see the [examples](#examples) below). +{{site.data.alerts.end}} + +### General + +Flag | Description +-----|----------- +`--attrs` | **Not relevant for single-node clusters.** Arbitrary strings, separated by colons, specifying node capability, which might include specialized hardware or number of cores, for example:

`--attrs=ram:64gb`

These can be used to influence the location of data replicas. See [Configure Replication Zones](configure-replication-zones.html#replication-constraints) for full details. +`--background` | Set this to start the node in the background. This is better than appending `&` to the command because control is returned to the shell only once the node is ready to accept requests.

**Note:** `--background` is suitable for writing automated test suites or maintenance procedures that need a temporary server process running in the background. It is not intended to be used to start a long-running server, because it does not fully detach from the controlling terminal. Consider using a service manager or a tool like [daemon(8)](https://www.freebsd.org/cgi/man.cgi?query=daemon&sektion=8) instead. +`--cache` | The total size for caches, shared evenly if there are multiple storage devices. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--cache=.25`
`--cache=25%`
`--cache=1000000000 ----> 1000000000 bytes`
`--cache=1GB ----> 1000000000 bytes`
`--cache=1GiB ----> 1073741824 bytes`

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead.

**Default:** `128MiB`

The default cache size is reasonable for local development clusters. For production deployments, this should be increased to 25% or higher. Increasing the cache size will generally improve the node's read performance. See [Recommended Production Settings](recommended-production-settings.html#cache-and-sql-memory-size) for more details. +`--external-io-dir` | The path of the external IO directory with which the local file access paths are prefixed while performing backup and restore operations using local node directories or NFS drives. If set to `disabled`, backups and restores using local node directories and NFS drives are disabled.

**Default:** `extern` subdirectory of the first configured [`store`](#store).

To set the `--external-io-dir` flag to the locations you want to use without needing to restart nodes, create symlinks to the desired locations from within the `extern` directory. +`--listening-url-file` | The file to which the node's SQL connection URL will be written on successful startup, in addition to being printed to the [standard output](#standard-output).

This is particularly helpful in identifying the node's port when an unused port is assigned automatically (`--port=0`). +`--locality` | **Not relevant for single-node clusters.** Arbitrary key-value pairs that describe the location of the node. Locality might include country, region, datacenter, rack, etc. For more details, see [Locality](cockroach-start.html#locality) below. +`--max-disk-temp-storage` | The maximum on-disk storage capacity available to store temporary data for SQL queries that exceed the memory budget (see `--max-sql-memory`). This ensures that JOINs, sorts, and other memory-intensive SQL operations are able to spill intermediate results to disk. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit (e.g., `.25`, `25%`, `500GB`, `1TB`, `1TiB`).

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead. Also, if expressed as a percentage, this value is interpreted relative to the size of the first store. However, the temporary space usage is never counted towards any store usage; therefore, when setting this value, it's important to ensure that the size of this temporary storage plus the size of the first store doesn't exceed the capacity of the storage device.

The temporary files are located in the path specified by the `--temp-dir` flag, or in the subdirectory of the first store (see `--store`) by default.

**Default:** `32GiB` +`--max-sql-memory` | The maximum in-memory storage capacity available to store temporary data for SQL queries, including prepared queries and intermediate data rows during query execution. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--max-sql-memory=.25`
`--max-sql-memory=25%`
`--max-sql-memory=10000000000 ----> 1000000000 bytes`
`--max-sql-memory=1GB ----> 1000000000 bytes`
`--max-sql-memory=1GiB ----> 1073741824 bytes`

The temporary files are located in the path specified by the `--temp-dir` flag, or in the subdirectory of the first store (see `--store`) by default.

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead.

**Default:** `25%`

The default SQL memory size is suitable for production deployments but can be raised to increase the number of simultaneous client connections the node allows as well as the node's capacity for in-memory processing of rows when using `ORDER BY`, `GROUP BY`, `DISTINCT`, joins, and window functions. For local development clusters with memory-intensive workloads, reduce this value to, for example, `128MiB` to prevent out of memory errors. +`--pid-file` | The file to which the node's process ID will be written on successful startup. When this flag is not set, the process ID is not written to file. +`--store`
`-s` | The file path to a storage device and, optionally, store attributes and maximum size. When using multiple storage devices for a node, this flag must be specified separately for each device, for example:

`--store=/mnt/ssd01 --store=/mnt/ssd02`

For more details, see [Store](#store) below. +`--temp-dir` | The path of the node's temporary store directory. On node start up, the location for the temporary files is printed to the standard output.

**Default:** Subdirectory of the first [store](#store) + +### Networking + +Flag | Description +-----|----------- +`--listen-addr` | The IP address/hostname and port to listen on for connections from clients. For IPv6, use the notation `[...]`, e.g., `[::1]` or `[fe80::f6f2:::]`.

**Default:** Listen on all IP addresses on port `26257` +`--http-addr` | The IP address/hostname and port to listen on for Admin UI HTTP requests. For IPv6, use the notation `[...]`, e.g., `[::1]:8080` or `[fe80::f6f2:::]:8080`.

**Default:** Listen on the address part of `--listen-addr` on port `8080` +`--socket-dir` | The directory path on which to listen for [Unix domain socket](https://en.wikipedia.org/wiki/Unix_domain_socket) connections from clients installed on the same Unix-based machine. For an example, see [Connect to a cluster listening for Unix domain socket connections](cockroach-sql.html#connect-to-a-cluster-listening-for-unix-domain-socket-connections). + +### Security + +Flag | Description +-----|----------- +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html). The directory must contain valid certificates if running in secure mode.

**Default:** `${HOME}/.cockroach-certs/` +`--insecure` | Run in insecure mode. If this flag is not set, the `--certs-dir` flag must point to valid certificates.

Note the following risks: An insecure cluster is open to any client that can access any node's IP addresses; any user, even `root`, can log in without providing a password; any user, connecting as `root`, can read or write any data in your cluster; and there is no network encryption or authentication, and thus no confidentiality.

**Default:** `false` +`--cert-principal-map` | A comma-separated list of `cert-principal:db-principal` mappings used to map the certificate principals to IP addresses, DNS names, and SQL users. This allows the use of certificates generated by Certificate Authorities that place restrictions on the contents of the `commonName` field. For usage information, see [Create Security Certificates using Openssl](create-security-certificates-openssl.html#examples). +`--enterprise-encryption` | This optional flag specifies the encryption options for one of the stores on the node. If multiple stores exist, the flag must be specified for each store.

This flag takes a number of options. For a complete list of options, and usage instructions, see [Encryption at Rest](encryption.html).

Note that this is an [enterprise feature](enterprise-licensing.html). + +### Store + +The `--store` flag supports the following fields. Note that commas are used to separate fields, and so are forbidden in all field values. + +{{site.data.alerts.callout_info}} +In-memory storage is not suitable for production deployments at this time. +{{site.data.alerts.end}} + +Field | Description +------|------------ +`type` | For in-memory storage, set this field to `mem`; otherwise, leave this field out. The `path` field must not be set when `type=mem`. +`path` | The file path to the storage device. When not setting `attr` or `size`, the `path` field label can be left out:

`--store=/mnt/ssd01`

When either of those fields are set, however, the `path` field label must be used:

`--store=path=/mnt/ssd01,size=20GB`

**Default:** `cockroach-data` +`attrs` | Arbitrary strings, separated by colons, specifying disk type or capability. These can be used to influence the location of data replicas. See [Configure Replication Zones](configure-replication-zones.html#replication-constraints) for full details.

In most cases, node-level `--locality` or `--attrs` are preferable to store-level attributes, but this field can be used to match capabilities for storage of individual databases or tables. For example, an OLTP database would probably want to allocate space for its tables only on solid state devices, whereas append-only time series might prefer cheaper spinning drives. Typical attributes include whether the store is flash (`ssd`) or spinny disk (`hdd`), as well as speeds and other specs, for example:

`--store=path=/mnt/hda1,attrs=hdd:7200rpm` +`size` | The maximum size allocated to the node. When this size is reached, CockroachDB attempts to rebalance data to other nodes with available capacity. When there's no capacity elsewhere, this limit will be exceeded. Also, data may be written to the node faster than the cluster can rebalance it away; in this case, as long as capacity is available elsewhere, CockroachDB will gradually rebalance data down to the store limit.

The `size` can be specified either in a bytes-based unit or as a percentage of hard drive space (notated as a decimal or with `%`), for example:

`--store=path=/mnt/ssd01,size=10000000000 ----> 10000000000 bytes`
`--store=path=/mnt/ssd01,size=20GB ----> 20000000000 bytes`
`--store=path=/mnt/ssd01,size=20GiB ----> 21474836480 bytes`
`--store=path=/mnt/ssd01,size=0.02TiB ----> 21474836480 bytes`
`--store=path=/mnt/ssd01,size=20% ----> 20% of available space`
`--store=path=/mnt/ssd01,size=0.2 ----> 20% of available space`
`--store=path=/mnt/ssd01,size=.2 ----> 20% of available space`

**Default:** 100%

For an in-memory store, the `size` field is required and must be set to the true maximum bytes or percentage of available memory, for example:

`--store=type=mem,size=20GB`
`--store=type=mem,size=90%`

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead. + +### Logging + +By default, `cockroach start-single-node` writes all messages to log files, and prints nothing to `stderr`. However, you can control the process's [logging](debug-and-error-logs.html) behavior with the following flags: + +{% include {{ page.version.version }}/misc/logging-flags.md %} + +#### Defaults + +`cockroach start-single-node` uses the equivalent values for these logging flags by default: + +- `--log-dir=/logs` +- `--logtostderr=NONE` + +This means, by default, CockroachDB writes all messages to log files, and never prints to `stderr`. + +## Standard output + +When you run `cockroach start-single-node`, some helpful details are printed to the standard output: + +~~~ shell +CockroachDB node starting at {{page.release_info.start_time}} +build: CCL {{page.release_info.version}} @ {{page.release_info.build_time}} (go1.12.6) +webui: http://localhost:8080 +sql: postgresql://root@localhost:26257?sslmode=disable +RPC client flags: cockroach --host=localhost:26257 --insecure +logs: /Users//node1/logs +temp dir: /Users//node1/cockroach-temp242232154 +external I/O path: /Users//node1/extern +store[0]: path=/Users//node1 +status: initialized new cluster +clusterID: 8a681a16-9623-4fc1-a537-77e9255daafd +nodeID: 1 +~~~ + +{{site.data.alerts.callout_success}} +These details are also written to the `INFO` log in the `/logs` directory. You can retrieve them with a command like `grep 'node starting' node1/logs/cockroach.log -A 11`. +{{site.data.alerts.end}} + +Field | Description +------|------------ +`build` | The version of CockroachDB you are running. +`webui` | The URL for accessing the Admin UI. +`sql` | The connection URL for your client. +`RPC client flags` | The flags to use when connecting to the node via [`cockroach` client commands](../cockroach-commands.html). +`logs` | The directory containing debug log data. +`temp dir` | The temporary store directory of the node. +`external I/O path` | The external IO directory with which the local file access paths are prefixed while performing [backup](backup.html) and [restore](restore.html) operations using local node directories or NFS drives. +`attrs` | If node-level attributes were specified in the `--attrs` flag, they are listed in this field. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`locality` | If values describing the locality of the node were specified in the `--locality` field, they are listed in this field. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`store[n]` | The directory containing store data, where `[n]` is the index of the store, e.g., `store[0]` for the first store, `store[1]` for the second store.

If store-level attributes were specified in the `attrs` field of the [`--store`](#store) flag, they are listed in this field as well. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`status` | Whether the node is the first in the cluster (`initialized new cluster`), joined an existing cluster for the first time (`initialized new node, joined pre-existing cluster`), or rejoined an existing cluster (`restarted pre-existing node`). +`clusterID` | The ID of the cluster.

When trying to join a node to an existing cluster, if this ID is different than the ID of the existing cluster, the node has started a new cluster. This may be due to conflicting information in the node's data directory. For additional guidance, see the [troubleshooting](common-errors.html#node-belongs-to-cluster-cluster-id-but-is-attempting-to-connect-to-a-gossip-network-for-cluster-another-cluster-id) docs. +`nodeID` | The ID of the node. +`socket` | If the `--socket-dir` flag specifies a directory on which to listen for Unix domain socket connections, this field lists the name of the socket file. + +## Examples + +### Start a single-node cluster + +
+ + +
+ +
+1. Create two directories for certificates: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs my-safe-directory + ~~~ + + Directory | Description + ----------|------------ + `certs` | You'll generate your CA certificate and all node and client certificates and keys in this directory. + `my-safe-directory` | You'll generate your CA key in this directory and then reference the key when generating node and client certificates. + +2. Create the CA (Certificate Authority) certificate and key pair: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +3. Create the certificate and key pair for the node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + localhost \ + $(hostname) \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +4. Create a client certificate and key pair for the `root` user: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-client \ + root \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +5. Start the single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node \ + --certs-dir=certs \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --background + ~~~ +
+ +
+

+{% include copy-clipboard.html %} +~~~ shell +$ cockroach start-single-node \ +--insecure \ +--listen-addr=localhost:26257 \ +--http-addr=localhost:8080 \ +--background +~~~ +
+ +### Scale to multiple nodes + +Scaling a cluster started with `cockroach start-single-node` involves restarting the first node with the `cockroach start` command instead, and then adding new nodes with that command as well, all using a `--join` flag that forms them into a single multi-node cluster. Since replication is disabled in clusters started with `start-single-node`, you also need to enable replication to get CockroachDB's availability and consistency guarantees. + +
+ + +
+ +
+1. Stop the single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit \ + --certs-dir=certs \ + --host=localhost:26257 + ~~~ + +2. Restart the node with the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + The new flag to note is `--join`, which specifies the addresses and ports of the nodes that will comprise your cluster. You'll use this exact `--join` flag when starting other nodes as well. + +3. Add two more nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --store=node2 \ + --listen-addr=localhost:26258 \ + --http-addr=localhost:8081 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --store=node3 \ + --listen-addr=localhost:26259 \ + --http-addr=localhost:8082 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + These commands are the same as before but with unique `--store`, `--listen-addr`, and `--http-addr` flags, since this all nodes are running on the same machine. Also, since all nodes use the same hostname (`localhost`), you can use the first node's certificate. Note that this is different than running a production cluster, where you would need to generate a certificate and key for each node, issued to all common names and IP addresses you might use to refer to the node as well as to any load balancer instances. + +4. Open the [built-in SQL shell](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs --host=localhost:26257 + ~~~ + +5. Update preconfigured [replication zones](configure-replication-zones.html) to replicate user data 3 times and import internal data 5 times: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE default CONFIGURE ZONE USING num_replicas = 3; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE system CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER database system CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE liveness CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE meta CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.jobs CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.replication_constraint_stats CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.replication_stats CONFIGURE ZONE USING num_replicas = 5; + ~~~ +
+ +
+1. Stop the single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit \ + --insecure \ + --host=localhost:26257 + ~~~ + +2. Restart the node with the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + The new flag to note is `--join`, which specifies the addresses and ports of the nodes that will comprise your cluster. You'll use this exact `--join` flag when starting other nodes as well. + +3. Add two more nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=node2 \ + --listen-addr=localhost:26258 \ + --http-addr=localhost:8081 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=node3 \ + --listen-addr=localhost:26259 \ + --http-addr=localhost:8082 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + These commands are the same as before but with unique `--store`, `--listen-addr`, and `--http-addr` flags, since this all nodes are running on the same machine. + +4. Open the [built-in SQL shell](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost:26257 + ~~~ + +5. Update preconfigured [replication zones](configure-replication-zones.html) to replicate user data 3 times and import internal data 5 times: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE default CONFIGURE ZONE USING num_replicas = 3; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE system CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER database system CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE liveness CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE meta CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.jobs CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.replication_constraint_stats CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE system.public.replication_stats CONFIGURE ZONE USING num_replicas = 5; + ~~~ +
+ +## See also + +- Running a local multi-node cluster: + - [From Binary](start-a-local-cluster.html) + - [In Kubernetes](orchestrate-a-local-cluster-with-kubernetes.html) + - [In Docker](start-a-local-cluster-in-docker.html) +- Running a distributed multi-node cluster: + - [From Binary](manual-deployment.html) + - [In Kubernetes](orchestrate-cockroachdb-with-kubernetes.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-start.md b/v20.2/cockroach-start.md new file mode 100644 index 00000000000..ad79e21c4f9 --- /dev/null +++ b/v20.2/cockroach-start.md @@ -0,0 +1,494 @@ +--- +title: cockroach start +summary: Start a new multi-node cluster or add nodes to an existing multi-node cluster. +toc: true +redirect_from: start-a-node.html +key: start-a-node.html +--- + +This page explains the `cockroach start` [command](cockroach-commands.html), which you use to start a new multi-node cluster or add nodes to an existing cluster. + +{{site.data.alerts.callout_success}} +If you need a simple single-node backend for app development, use [`cockroach start-single-node`](cockroach-start-single-node.html) instead. For quick SQL testing, consider using [`cockroach demo`](cockroach-demo.html) to start a temporary, in-memory cluster with immediate access to an interactive SQL shell. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +Node-level settings are defined by [flags](#flags) passed to the `cockroach start` command and cannot be changed without stopping and restarting the node. In contrast, some cluster-wide settings are defined via SQL statements and can be updated anytime after a cluster has been started. For more details, see [Cluster Settings](cluster-settings.html). +{{site.data.alerts.end}} + +## Synopsis + +Start a node to be part of a new multi-node cluster: + +~~~ shell +$ cockroach start +~~~ + +Initialize a new multi-node cluster: + +~~~ shell +$ cockroach init +~~~ + +Add a node to an existing cluster: + +~~~ shell +$ cockroach start +~~~ + +View help: + +~~~ shell +$ cockroach start --help +~~~ + +## Flags + +The `cockroach start` command supports the following [general-use](#general), [networking](#networking), [security](#security), and [logging](#logging) flags. + +Many flags have useful defaults that can be overridden by specifying the flags explicitly. If you specify flags explicitly, however, be sure to do so each time the node is restarted, as they will not be remembered. The one exception is the `--join` flag, which is stored in a node's data directory, but even for `--join`, it's best practices to specify the flag every time, as that will allow restarted nodes to join the cluster even if their data directory was destroyed. + +### General + +Flag | Description +-----|----------- +`--attrs` | Arbitrary strings, separated by colons, specifying node capability, which might include specialized hardware or number of cores, for example:

`--attrs=ram:64gb`

These can be used to influence the location of data replicas. See [Configure Replication Zones](configure-replication-zones.html#replication-constraints) for full details. +`--background` | Set this to start the node in the background. This is better than appending `&` to the command because control is returned to the shell only once the node is ready to accept requests.

**Note:** `--background` is suitable for writing automated test suites or maintenance procedures that need a temporary server process running in the background. It is not intended to be used to start a long-running server, because it does not fully detach from the controlling terminal. Consider using a service manager or a tool like [daemon(8)](https://www.freebsd.org/cgi/man.cgi?query=daemon&sektion=8) instead. +`--cache` | The total size for caches, shared evenly if there are multiple storage devices. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--cache=.25`
`--cache=25%`
`--cache=1000000000 ----> 1000000000 bytes`
`--cache=1GB ----> 1000000000 bytes`
`--cache=1GiB ----> 1073741824 bytes`

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead.

**Default:** `128MiB`

The default cache size is reasonable for local development clusters. For production deployments, this should be increased to 25% or higher. Increasing the cache size will generally improve the node's read performance. See [Recommended Production Settings](recommended-production-settings.html#cache-and-sql-memory-size) for more details. +`--external-io-dir` | The path of the external IO directory with which the local file access paths are prefixed while performing backup and restore operations using local node directories or NFS drives. If set to `disabled`, backups and restores using local node directories and NFS drives, as well as [`cockroach nodelocal upload`](cockroach-nodelocal-upload.html), are disabled.

**Default:** `extern` subdirectory of the first configured [`store`](#store).

To set the `--external-io-dir` flag to the locations you want to use without needing to restart nodes, create symlinks to the desired locations from within the `extern` directory. +`--listening-url-file` | The file to which the node's SQL connection URL will be written as soon as the node is ready to accept connections, in addition to being printed to the [standard output](#standard-output). When `--background` is used, this happens before the process detaches from the terminal.

This is particularly helpful in identifying the node's port when an unused port is assigned automatically (`--port=0`). +`--locality` | Arbitrary key-value pairs that describe the location of the node. Locality might include country, region, datacenter, rack, etc. For more details, see [Locality](#locality) below. +`--max-disk-temp-storage` | The maximum on-disk storage capacity available to store temporary data for SQL queries that exceed the memory budget (see `--max-sql-memory`). This ensures that JOINs, sorts, and other memory-intensive SQL operations are able to spill intermediate results to disk. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit (e.g., `.25`, `25%`, `500GB`, `1TB`, `1TiB`).

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead. Also, if expressed as a percentage, this value is interpreted relative to the size of the first store. However, the temporary space usage is never counted towards any store usage; therefore, when setting this value, it's important to ensure that the size of this temporary storage plus the size of the first store doesn't exceed the capacity of the storage device.

The temporary files are located in the path specified by the `--temp-dir` flag, or in the subdirectory of the first store (see `--store`) by default.

**Default:** `32GiB` +`--max-offset` | The maximum allowed clock offset for the cluster. If observed clock offsets exceed this limit, servers will crash to minimize the likelihood of reading inconsistent data. Increasing this value will increase the time to recovery of failures as well as the frequency of uncertainty-based read restarts.

Note that this value must be the same on all nodes in the cluster and cannot be changed with a [rolling upgrade](upgrade-cockroach-version.html). In order to change it, first stop every node in the cluster. Then once the entire cluster is offline, restart each node with the new value.

**Default:** `500ms` +`--max-sql-memory` | The maximum in-memory storage capacity available to store temporary data for SQL queries, including prepared queries and intermediate data rows during query execution. This can be a percentage (notated as a decimal or with `%`) or any bytes-based unit, for example:

`--max-sql-memory=.25`
`--max-sql-memory=25%`
`--max-sql-memory=10000000000 ----> 1000000000 bytes`
`--max-sql-memory=1GB ----> 1000000000 bytes`
`--max-sql-memory=1GiB ----> 1073741824 bytes`

The temporary files are located in the path specified by the `--temp-dir` flag, or in the subdirectory of the first store (see `--store`) by default.

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead.

**Default:** `25%`

The default SQL memory size is suitable for production deployments but can be raised to increase the number of simultaneous client connections the node allows as well as the node's capacity for in-memory processing of rows when using `ORDER BY`, `GROUP BY`, `DISTINCT`, joins, and window functions. For local development clusters with memory-intensive workloads, reduce this value to, for example, `128MiB` to prevent out of memory errors. +`--pid-file` | The file to which the node's process ID will be written as soon as the node is ready to accept connections. When `--background` is used, this happens before the process detaches from the terminal. When this flag is not set, the process ID is not written to file. +`--store`
`-s` | The file path to a storage device and, optionally, store attributes and maximum size. When using multiple storage devices for a node, this flag must be specified separately for each device, for example:

`--store=/mnt/ssd01 --store=/mnt/ssd02`

For more details, see [Store](#store) below. +`--temp-dir` | The path of the node's temporary store directory. On node start up, the location for the temporary files is printed to the standard output.

**Default:** Subdirectory of the first [store](#store) + +### Networking + +Flag | Description +-----|----------- +`--advertise-addr` | The IP address/hostname and port to tell other nodes to use. If using a hostname, it must be resolvable from all nodes. If using an IP address, it must be routable from all nodes; for IPv6, use the notation `[...]`, e.g., `[::1]` or `[fe80::f6f2:::]`.

This flag's effect depends on how it is used in combination with `--listen-addr`. For example, if the port number is different than the one used in `--listen-addr`, port forwarding is required. For more details, see [Networking](recommended-production-settings.html#networking).

**Default:** The value of `--listen-addr`; if `--listen-addr` is not specified, advertises the node's canonical hostname and port `26257` +`--listen-addr` | The IP address/hostname and port to listen on for connections from other nodes and clients. For IPv6, use the notation `[...]`, e.g., `[::1]` or `[fe80::f6f2:::]`.

This flag's effect depends on how it is used in combination with `--advertise-addr`. For example, the node will also advertise itself to other nodes using this value if `--advertise-addr` is not specified. For more details, see [Networking](recommended-production-settings.html#networking).

**Default:** Listen on all IP addresses on port `26257`; if `--advertise-addr` is not specified, also advertise the node's canonical hostname to other nodes +`--http-addr` | The IP address/hostname and port to listen on for Admin UI HTTP requests. For IPv6, use the notation `[...]`, e.g., `[::1]:8080` or `[fe80::f6f2:::]:8080`.

**Default:** Listen on the address part of `--listen-addr` on port `8080` +`--locality-advertise-addr` | The IP address/hostname and port to tell other nodes in specific localities to use. This flag is useful when running a cluster across multiple networks, where nodes in a given network have access to a private or local interface while nodes outside the network do not. In this case, you can use `--locality-advertise-addr` to tell nodes within the same network to prefer the private or local address to improve performance and use `--advertise-addr` to tell nodes outside the network to use another address that is reachable from them.

This flag relies on nodes being started with the [`--locality`](#locality) flag and uses the `locality@address` notation, for example:

`--locality-advertise-addr=region=us-west@10.0.0.0:26257`

See the [example](#start-a-multi-node-cluster-across-private-networks) below for more details. +`--join`
`-j` | The addresses for connecting the node to a cluster.

When starting a multi-node cluster for the first time, set this flag to the addresses of 3-5 of the initial nodes. Then run the [`cockroach init`](cockroach-init.html) command against any of the nodes to complete cluster startup. See the [example](#start-a-multi-node-cluster) below for more details.

When adding a node to an existing cluster, set this flag to 3-5 of the nodes already in the cluster; it's easiest to use the same list of addresses that was used to start the initial nodes.

Changed in v19.2: Running `cockroach start` without the `--join` flag has been deprecated. To start a single-node cluster, use `cockroach start-single-node` instead. +`--socket-dir` | The directory path on which to listen for [Unix domain socket](https://en.wikipedia.org/wiki/Unix_domain_socket) connections from clients installed on the same Unix-based machine. For an example, see [Connect to a cluster listening for Unix domain socket connections](cockroach-sql.html#connect-to-a-cluster-listening-for-unix-domain-socket-connections). +`--advertise-host` | **Deprecated.** Use `--advertise-addr` instead. +`--host` | **Deprecated.** Use `--listen-addr` instead. +`--port`
`-p` | **Deprecated.** Specify port in `--advertise-addr` and/or `--listen-addr` instead. +`--http-host` | **Deprecated.** Use `--http-addr` instead. +`--http-port` | **Deprecated.** Specify port in `--http-addr` instead. + +### Security + +Flag | Description +-----|----------- +`--certs-dir` | The path to the [certificate directory](cockroach-cert.html). The directory must contain valid certificates if running in secure mode.

**Default:** `${HOME}/.cockroach-certs/` +`--insecure` | Run in insecure mode. If this flag is not set, the `--certs-dir` flag must point to valid certificates.

Note the following risks: An insecure cluster is open to any client that can access any node's IP addresses; any user, even `root`, can log in without providing a password; any user, connecting as `root`, can read or write any data in your cluster; and there is no network encryption or authentication, and thus no confidentiality.

**Default:** `false` +`--cert-principal-map` | A comma-separated list of `cert-principal:db-principal` mappings used to map the certificate principals to IP addresses, DNS names, and SQL users. This allows the use of certificates generated by Certificate Authorities that place restrictions on the contents of the `commonName` field. For usage information, see [Create Security Certificates using Openssl](create-security-certificates-openssl.html#examples). +`--enterprise-encryption` | This optional flag specifies the encryption options for one of the stores on the node. If multiple stores exist, the flag must be specified for each store.

This flag takes a number of options. For a complete list of options, and usage instructions, see [Encryption at Rest](encryption.html).

Note that this is an [enterprise feature](enterprise-licensing.html). +`--external-io-disable-http` | This optional flag disables external HTTP(S) access (as well as custom HTTP(S) endpoints) when performing bulk operations (e.g, [`BACKUP`](backup.html), [`IMPORT`](import.html), etc.). This can be used in environments where you cannot run a full proxy server.

If you want to run a proxy server, you can start CockroachDB while specifying the `HTTP(S)_PROXY` environment variable. +`--external-io-disable-implicit-credentials` | This optional flag disables the use of implicit credentials when accessing external cloud storage services for bulk operations (e.g, [`BACKUP`](backup.html), [`IMPORT`](import.html), etc.). + +### Locality + +The `--locality` flag accepts arbitrary key-value pairs that describe the location of the node. Locality might include region, country, datacenter, rack, etc. The key-value pairs should be ordered into _locality tiers_ from most inclusive to least inclusive (e.g., region before datacenter as in `region=eu,dc=paris`), and the keys and order of key-value pairs must be the same on all nodes. It's typically better to include more pairs than fewer. + +- CockroachDB spreads the replicas of each piece of data across as diverse a set of localities as possible, with the order determining the priority. Locality can also be used to influence the location of data replicas in various ways using [replication zones](configure-replication-zones.html#replication-constraints). + +- When there is high latency between nodes (e.g., cross-datacenter deployments), CockroachDB uses locality to move range leases closer to the current workload, reducing network round trips and improving read performance, also known as ["follow-the-workload"](demo-follow-the-workload.html). In a deployment across more than 3 datacenters, however, to ensure that all data benefits from "follow-the-workload", you must increase your replication factor to match the total number of datacenters. + +- Locality is also a prerequisite for using the [table partitioning](partitioning.html) and [**Node Map**](enable-node-map.html) enterprise features. + +#### Example + +~~~ shell +# Locality flag for nodes in US East datacenter: +--locality=region=us,datacenter=us-east + +# Locality flag for nodes in US Central datacenter: +--locality=region=us,datacenter=us-central + +# Locality flag for nodes in US West datacenter: +--locality=region=us,datacenter=us-west +~~~ + +### Storage + +#### Storage engine + + The `--storage-engine` flag is used to choose the storage engine used by the node. Note that this setting applies to all [stores](#store) on the node, including the [temp store](#temp-dir). + +Supported options: + +- `default`: Checks which engine type was last used for this node's [store directory](#store) (Pebble or RocksDB), and uses that engine. If more than one store is specified, the previous engine type of the first store is used. If the check fails for any reason, or if the store directory does not exist yet, RocksDB is used. +- `rocksdb`: Uses the [RocksDB](https://rocksdb.org) storage engine. +- `pebble`: Uses the experimental [Pebble storage engine](https://github.com/cockroachdb/pebble). Pebble is intended to be bidirectionally compatible with the RocksDB on-disk format. Pebble differs from RocksDB in that it is written in Go and implements a subset of RocksDB's large feature set. + +#### Store + +The `--store` flag supports the following fields. Note that commas are used to separate fields, and so are forbidden in all field values. + +{{site.data.alerts.callout_info}} +In-memory storage is not suitable for production deployments at this time. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/misc/multi-store-nodes.md %} + +Field | Description +------|------------ +`type` | For in-memory storage, set this field to `mem`; otherwise, leave this field out. The `path` field must not be set when `type=mem`. +`path` | The file path to the storage device. When not setting `attr` or `size`, the `path` field label can be left out:

`--store=/mnt/ssd01`

When either of those fields are set, however, the `path` field label must be used:

`--store=path=/mnt/ssd01,size=20GB`

**Default:** `cockroach-data` +`attrs` | Arbitrary strings, separated by colons, specifying disk type or capability. These can be used to influence the location of data replicas. See [Configure Replication Zones](configure-replication-zones.html#replication-constraints) for full details.

In most cases, node-level `--locality` or `--attrs` are preferable to store-level attributes, but this field can be used to match capabilities for storage of individual databases or tables. For example, an OLTP database would probably want to allocate space for its tables only on solid state devices, whereas append-only time series might prefer cheaper spinning drives. Typical attributes include whether the store is flash (`ssd`) or spinny disk (`hdd`), as well as speeds and other specs, for example:

`--store=path=/mnt/hda1,attrs=hdd:7200rpm` +`size` | The maximum size allocated to the node. When this size is reached, CockroachDB attempts to rebalance data to other nodes with available capacity. When there's no capacity elsewhere, this limit will be exceeded. Also, data may be written to the node faster than the cluster can rebalance it away; in this case, as long as capacity is available elsewhere, CockroachDB will gradually rebalance data down to the store limit.

The `size` can be specified either in a bytes-based unit or as a percentage of hard drive space (notated as a decimal or with `%`), for example:

`--store=path=/mnt/ssd01,size=10000000000 ----> 10000000000 bytes`
`--store=path=/mnt/ssd01,size=20GB ----> 20000000000 bytes`
`--store=path=/mnt/ssd01,size=20GiB ----> 21474836480 bytes`
`--store=path=/mnt/ssd01,size=0.02TiB ----> 21474836480 bytes`
`--store=path=/mnt/ssd01,size=20% ----> 20% of available space`
`--store=path=/mnt/ssd01,size=0.2 ----> 20% of available space`
`--store=path=/mnt/ssd01,size=.2 ----> 20% of available space`

**Default:** 100%

For an in-memory store, the `size` field is required and must be set to the true maximum bytes or percentage of available memory, for example:

`--store=type=mem,size=20GB`
`--store=type=mem,size=90%`

Note: If you use the `%` notation, you might need to escape the `%` sign, for instance, while configuring CockroachDB through `systemd` service files. For this reason, it's recommended to use the decimal notation instead. + +### Logging + +By default, `cockroach start` writes all messages to log files, and prints nothing to `stderr`. However, you can control the process's [logging](debug-and-error-logs.html) behavior with the following flags: + +{% include {{ page.version.version }}/misc/logging-flags.md %} + +#### Defaults + +`cockroach start` uses the equivalent values for these logging flags by default: + +- `--log-dir=/logs` +- `--logtostderr=NONE` + +This means, by default, CockroachDB writes all messages to log files, and never prints to `stderr`. + +## Standard output + +When you run `cockroach start`, some helpful details are printed to the standard output: + +~~~ shell +CockroachDB node starting at {{page.release_info.start_time}} +build: CCL {{page.release_info.version}} @ {{page.release_info.build_time}} (go1.12.6) +webui: http://localhost:8080 +sql: postgresql://root@localhost:26257?sslmode=disable +RPC client flags: cockroach --host=localhost:26257 --insecure +logs: /Users//node1/logs +temp dir: /Users//node1/cockroach-temp242232154 +external I/O path: /Users//node1/extern +store[0]: path=/Users//node1 +status: initialized new cluster +clusterID: 8a681a16-9623-4fc1-a537-77e9255daafd +nodeID: 1 +~~~ + +{{site.data.alerts.callout_success}} +These details are also written to the `INFO` log in the `/logs` directory. You can retrieve them with a command like `grep 'node starting' node1/logs/cockroach.log -A 11`. +{{site.data.alerts.end}} + +Field | Description +------|------------ +`build` | The version of CockroachDB you are running. +`webui` | The URL for accessing the Admin UI. +`sql` | The connection URL for your client. +`RPC client flags` | The flags to use when connecting to the node via [`cockroach` client commands](../cockroach-commands.html). +`logs` | The directory containing debug log data. +`temp dir` | The temporary store directory of the node. +`external I/O path` | The external IO directory with which the local file access paths are prefixed while performing [backup](backup.html) and [restore](restore.html) operations using local node directories or NFS drives. +`attrs` | If node-level attributes were specified in the `--attrs` flag, they are listed in this field. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`locality` | If values describing the locality of the node were specified in the `--locality` field, they are listed in this field. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`store[n]` | The directory containing store data, where `[n]` is the index of the store, e.g., `store[0]` for the first store, `store[1]` for the second store.

If store-level attributes were specified in the `attrs` field of the [`--store`](#store) flag, they are listed in this field as well. These details are potentially useful for [configuring replication zones](configure-replication-zones.html). +`status` | Whether the node is the first in the cluster (`initialized new cluster`), joined an existing cluster for the first time (`initialized new node, joined pre-existing cluster`), or rejoined an existing cluster (`restarted pre-existing node`). +`clusterID` | The ID of the cluster.

When trying to join a node to an existing cluster, if this ID is different than the ID of the existing cluster, the node has started a new cluster. This may be due to conflicting information in the node's data directory. For additional guidance, see the [troubleshooting](common-errors.html#node-belongs-to-cluster-cluster-id-but-is-attempting-to-connect-to-a-gossip-network-for-cluster-another-cluster-id) docs. +`nodeID` | The ID of the node. +`socket` | If the `--socket-dir` flag specifies a directory on which to listen for Unix domain socket connections, this field lists the name of the socket file. + +## Known limitations + +{% include {{ page.version.version }}/known-limitations/adding-stores-to-node.md %} + +## Examples + +### Start a multi-node cluster + +
+ + +
+ +To start a multi-node cluster, run the `cockroach start` command for each node, setting the `--join` flag to the addresses of 3-5 of the initial nodes: + +
+ +{{site.data.alerts.callout_success}} +Before starting the cluster, use [`cockroach cert`](cockroach-cert.html) to generate node and client certificates for a secure cluster connection. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--certs-dir=certs \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--certs-dir=certs \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--certs-dir=certs \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ +
+ +
+{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ +
+ +Then run the [`cockroach init`](cockroach-init.html) command against any node to perform a one-time cluster initialization: + +
+{% include copy-clipboard.html %} +~~~ shell +$ cockroach init \ +--certs-dir=certs \ +--host=
+~~~ +
+ +
+{% include copy-clipboard.html %} +~~~ shell +$ cockroach init \ +--insecure \ +--host=
+~~~ +
+ +### Start a multi-node cluster across private networks + +**Scenario:** + +- You have a cluster that spans GCE and AWS. +- The nodes on each cloud can reach each other on private addresses, but the private addresses aren't reachable from the other cloud. + +**Approach:** + +1. Start each node on GCE with `--locality` set to describe its location, `--locality-advertise-addr` set to advertise its private address to other nodes in on GCE, `--advertise-addr` set to advertise its public address to nodes on AWS, and `--join` set to the public addresses of 3-5 of the initial nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --locality=cloud=gce \ + --locality-advertise-addr=cloud=gce@ \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 + ~~~ + +2. Start each node on AWS with `--locality` set to describe its location, `--locality-advertise-addr` set to advertise its private address to other nodes on AWS, `--advertise-addr` set to advertise its public address to nodes on GCE, and `--join` set to the public addresses of 3-5 of the initial nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --locality=cloud=aws \ + --locality-advertise-addr=cloud=aws@ \ + --advertise-addr= \ + --join=,, \ + --cache=.25 \ + --max-sql-memory=.25 + ~~~ + +3. Run the [`cockroach init`](cockroach-init.html) command against any node to perform a one-time cluster initialization: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init \ + --certs-dir=certs \ + --host=
+ ~~~ + +### Add a node to a cluster + +
+ + +
+ +To add a node to an existing cluster, run the `cockroach start` command, setting the `--join` flag to the addresses of 3-5 of the nodes already in the cluster: + +
+{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--certs-dir=certs \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ +
+ +
+{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--advertise-addr= \ +--join=,, \ +--cache=.25 \ +--max-sql-memory=.25 +~~~ +
+ +### Create a table with node locality information + +Start a three-node cluster with locality information specified in the `cockroach start` commands: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --insecure --port=26257 --http-port=26258 --store=cockroach-data/1 --cache=256MiB --locality=region=eu-west-1,cloud=aws,zone=eu-west-1a +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --insecure --port=26259 --http-port=26260 --store=cockroach-data/2 --cache=256MiB --join=localhost:26257 --locality=region=eu-west-1,cloud=aws,zone=eu-west-1b +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --insecure --port=26261 --http-port=26262 --store=cockroach-data/3 --cache=256MiB --join=localhost:26257 --locality=region=eu-west-1,cloud=aws,zone=eu-west-1c +~~~ + +You can use the [`crdb_internal.locality_value`](functions-and-operators.html#system-info-functions) built-in function to return the current node's locality information from inside a SQL shell. The example below uses the output of `crdb_internal.locality_value('zone')` as the `DEFAULT` value to use for the `zone` column of new rows. Other available locality keys for the running three-node cluster include `region` and `cloud`. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE charges ( + zone STRING NOT NULL DEFAULT crdb_internal.locality_value('zone'), + id INT PRIMARY KEY NOT NULL +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO charges (id) VALUES (1); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM charges WHERE id = 1; +~~~ + +~~~ + zone | id ++------------+----+ + eu-west-1a | 1 +(1 row) +~~~ + +The `zone ` column has the zone of the node on which the row was created. + +In a separate terminal window, open a SQL shell to a different node on the cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --port 26259 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO charges (id) VALUES (2); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM charges WHERE id = 2; +~~~ + +~~~ + zone | id ++------------+----+ + eu-west-1b | 2 +(1 row) +~~~ + +In a separate terminal window, open a SQL shell to the third node: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --port 26261 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO charges (id) VALUES (3); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM charges WHERE id = 3; +~~~ + +~~~ + zone | id ++------------+----+ + eu-west-1c | 3 +(1 row) +~~~ + + +## See also + +- [Initialize a Cluster](cockroach-init.html) +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Local Deployment](start-a-local-cluster.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-version.md b/v20.2/cockroach-version.md new file mode 100644 index 00000000000..d7e1233da62 --- /dev/null +++ b/v20.2/cockroach-version.md @@ -0,0 +1,43 @@ +--- +title: cockroach version +summary: To view version details for a specific cockroach binary, run the cockroach version command. +toc: false +redirect_from: view-version-details.html +key: view-version-details.html +--- + +To view version details for a specific `cockroach` binary, run the `cockroach version` [command](cockroach-commands.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach version +~~~ + +~~~ +Build Tag: {{page.release_info.version}} +Build Time: {{page.release_info.build_time}} +Distribution: CCL +Platform: darwin amd64 +Go Version: go1.8.3 +C Compiler: 4.2.1 Compatible Clang 3.8.0 (tags/RELEASE_380/final) +Build SHA-1: 5b757262d33d814bda1deb2af20161a1f7749df3 +Build Type: release +~~~ + +The `cockroach version` command outputs the following fields: + +Field | Description +------|------------ +`Build Tag` | The CockroachDB version. +`Build Time` | The date and time when the binary was built. +`Distribution` | The scope of the binary. If `CCL`, the binary contains functionality covered by both the CockroachDB Community License (CCL) and the Business Source License (BSL). If `OSS`, the binary contains only functionality covered by the Apache 2.0 license. The v19.2 release converts to Apache 2.0 as of Oct 1, 2022, at which time you can use the `make buildoss` command to build a pure open-source binary. For more details about licensing, see the [Licensing FAQs](licensing-faqs.html). +`Platform` | The platform that the binary can run on. +`Go Version` | The version of Go in which the source code is written. +`C Compiler` | The C compiler used to build the binary. +`Build SHA-1` | The SHA-1 hash of the commit used to build the binary. +`Build Type` | The type of release. If `release`, `release-gnu`, or `release-musl`, the binary is for a [production release](../releases/#production-releases). If `development`, the binary is for a [testing release](../releases/#testing-releases). + +## See also + +- [Install CockroachDB](install-cockroachdb.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/cockroach-workload.md b/v20.2/cockroach-workload.md new file mode 100644 index 00000000000..2231ad165c6 --- /dev/null +++ b/v20.2/cockroach-workload.md @@ -0,0 +1,662 @@ +--- +title: cockroach workload +summary: Use cockroach workload to run a load generator against a CockroachDB cluster. +toc: true +--- + +CockroachDB comes with built-in load generators for simulating different types of client workloads, printing per-operation statistics and totals after a specific duration or max number of operations. To run one of these load generators, use the `cockroach workload` [command](cockroach-commands.html) as described below. + +{{site.data.alerts.callout_danger}} +The `cockroach workload` command is experimental. The interface and output are subject to change. +{{site.data.alerts.end}} + +## Synopsis + +Create the schema for a workload: + +~~~ shell +$ cockroach workload init '' +~~~ + +Run a workload: + +~~~ shell +$ cockroach workload run '' +~~~ + +View help: + +~~~ shell +$ cockroach workload --help +~~~ +~~~ shell +$ cockroach workload init --help +~~~ +~~~ shell +$ cockroach workload init --help +~~~ +~~~ shell +$ cockroach workload run --help +~~~ +~~~ shell +$ cockroach workload run --help +~~~ + +## Subcommands + +Command | Usage +--------|------ +`init` | Load the schema for the workload. You run this command once for a given schema. +`run` | Run a workload. You can run this command multiple times from different machines to increase concurrency. See [Concurrency](#concurrency) for more details. + +## Concurrency + +There are two ways to increase the concurrency of a workload: + +- **Increase the concurrency of a single workload instance** by running `cockroach workload run ` with the `--concurrency` flag set to a value higher than the default. +- **Run multiple instances of a workload in parallel** by running `cockroach workload run ` multiple times from different machines. + +## Workloads + +Workload | Description +---------|------------ +[`bank`](#bank-workload) | Models a set of accounts with currency balances.

For this workload, you run `workload init` to load the schema and then `workload run` to generate data. +[`intro`](#intro-and-startrek-workloads) | Loads an `intro` database, with one table, `mytable`, with a hidden message.

For this workload, you run only `workload init` to load the data. The `workload run` subcommand is not applicable. +[`kv`](#kv-workload) | Reads and writes to keys spread (by default, uniformly at random) across the cluster.

For this workload, you run `workload init` to load the schema and then `workload run` to generate data. +[`movr`](#movr-workload) | Simulates a workload for the [MovR example application](movr.html).

For this workload, you run `workload init` to load the schema and then `workload run` to generate data. +[`startrek`](#intro-and-startrek-workloads) | Loads a `startrek` database, with two tables, `episodes` and `quotes`.

For this workload, you run only `workload init` to load the data. The `workload run` subcommand is not applicable. +[`tpcc`](#tpcc-workload) | Simulates a transaction processing workload using a rich schema of multiple tables.

For this workload, you run `workload init` to load the schema and then `workload run` to generate data. +[`ycsb`](#ycsb-workload) | Simulates a high-scale key value workload, either read-heavy, write-heavy, or scan-based, with additional customizations.

For this workload, you run `workload init` to load the schema and then `workload run` to generate data. + +{{site.data.alerts.callout_info}} + `cockroach workload` sets the [`application_name`](set-vars.html#supported-variables) for its workload queries to the name of the workload that is used. You can filter queries on `application_name` on the [Statements page of the Admin UI](admin-ui-statements-page.html#search-and-filter-by-application), or in a [`SHOW QUERIES`](show-queries.html#filter-for-specific-queries) statement. +{{site.data.alerts.end}} + +## Flags + +{{site.data.alerts.callout_info}} +The `cockroach workload` command does not support connection or security flags like other [`cockroach` commands](cockroach-commands.html). Instead, you must use a [connection string](connection-parameters.html) at the end of the command. +{{site.data.alerts.end}} + +### `bank` workload + +Flag | Description +-----|------------ +`--concurrency` | The number of concurrent workers.

**Applicable commands:** `init` or `run`
**Default:** 2 * number of CPUs +`--db` | The SQL database to use.

**Applicable commands:** `init` or `run`
**Default:** `bank` +`--display-every` | The frequency for printing per-operation statistics. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1s` +`--display-format` | The format for printing per-operation statistics (`simple`, `incremental-json`). When using `incremental-json`, note that totals are not printed at the end of the workload's duration.

**Applicable command:** `run`
**Default:** `simple` +`--drop` | Drop the existing database, if it exists.

**Applicable commands:** `init` or `run`. For the `run` command, this flag must be used in conjunction with `--init`. +`--duration` | The duration to run, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable commands:** `init` or `run`
**Default:** `0`, which means run forever. +`--histograms` | The file to write per-op incremental and cumulative histogram data to.

**Applicable command:** `run` +`--init` | **Deprecated.** Use the `init` command instead.

**Applicable command:** `run` +`--max-ops` | The maximum number of operations to run.

**Applicable command:** `run` +`--max-rate` | The maximum frequency of operations (reads/writes).

**Applicable command:** `run`
**Default:** `0`, which means unlimited. +`--payload-bytes` | The size of the payload field in each initial row.

**Applicable commands:** `init` or `run`
**Default:** `100` +`--ramp` | The duration over which to ramp up load.

**Applicable command:** `run` +`--ranges` | The initial number of ranges in the `bank` table.

**Applicable commands:** `init` or `run`
**Default:** `10` +`--rows` | The initial number of accounts in the `bank` table.

**Applicable commands:** `init` or `run`
**Default:** `1000` +`--seed` | The key hash seed.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--tolerate-errors` | Keep running on error.

**Applicable command:** `run` + +### `intro` and `startrek` workloads + +{{site.data.alerts.callout_info}} +These workloads generate data but do not offer the ability to run continuous load. Thus, only the `init` subcommand is supported. +{{site.data.alerts.end}} + +Flag | Description +-----|------------ +`--drop` | Drop the existing database, if it exists, before loading the dataset. + + +### `kv` workload + +Flag | Description +-----|------------ +`--batch` | The number of blocks to insert in a single SQL statement.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--concurrency` | The number of concurrent workers.

**Applicable commands:** `init` or `run`
**Default:** `8` `--cycle-length`| The number of keys repeatedly accessed by each writer.**Applicable commands:** `init` or `run`
**Default:** `9223372036854775807` +`--db` | The SQL database to use.

**Applicable commands:** `init` or `run`
**Default:** `kv` +`--display-every` | The frequency for printing per-operation statistics. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1s` +`--display-format` | The format for printing per-operation statistics (`simple`, `incremental-json`). When using `incremental-json`, note that totals are not printed at the end of the workload's duration.

**Applicable command:** `run`
**Default:** `simple` +`--drop` | Drop the existing database, if it exists.

**Applicable commands:** `init` or `run` +`--duration` | The duration to run, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `0`, which means run forever. +`--histograms` | The file to write per-op incremental and cumulative histogram data to.

**Applicable command:** `run` +`--init` | **Deprecated.** Use the `init` command instead.

**Applicable command:** `run` +`--max-block-bytes` | The maximum amount of raw data written with each insertion.

**Applicable commands:** `init` or `run`
**Default:** `2` +`--max-ops` | The maximum number of operations to run.

**Applicable command:** `run` +`--max-rate` | The maximum frequency of operations (reads/writes).

**Applicable command:** `run`
**Default:** `0`, which means unlimited. +`--min-block-bytes` | The minimum amount of raw data written with each insertion.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--ramp` | The duration over which to ramp up load.

**Applicable command:** `run` +`--read-percent` | The percent (0-100) of operations that are reads of existing keys.

**Applicable commands:** `init` or `run` +`--seed` | The key hash seed.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--sequential` | Pick keys sequentially instead of randomly.

**Applicable commands:** `init` or `run` +`--splits` | The number of splits to perform before starting normal operations.

**Applicable commands:** `init` or `run` +`--tolerate-errors` | Keep running on error.

**Applicable command:** `run` +`--use-opt` | Use [cost-based optimizer](cost-based-optimizer.html).

**Applicable commands:** `init` or `run`
**Default:** `true` +`--write-seq` | Initial write sequence value.

**Applicable commands:** `init` or `run` + +### `movr` workload + +Flag | Description +-----|------------ +`--concurrency` | The number of concurrent workers.

**Applicable commands:** `init` or `run`
**Default:** `16` +`--data-loader` | How to load initial table data. Valid options are `INSERT` and `IMPORT`.

**Applicable commands:** `init` or `run`
**Default:** `INSERT` +`--db` | The SQL database to use.

**Applicable commands:** `init` or `run`
**Default:** `movr` +`--display-every` | The frequency for printing per-operation statistics. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1s` +`--display-format` | The format for printing per-operation statistics (`simple`, `incremental-json`). When using `incremental-json`, note that totals are not printed at the end of the workload's duration.

**Applicable command:** `run`
**Default:** `simple` +`--drop` | Drop the existing database, if it exists.

**Applicable commands:** `init` or `run` +`--duration` | The duration to run, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `0`, which means run forever. +`--histograms` | The file to write per-op incremental and cumulative histogram data to.

**Applicable command:** `run` +`--histograms-max-latency` | Expected maximum latency of running a query, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1m40s` +`--max-ops` | The maximum number of operations to run.

**Applicable command:** `run` +`--max-rate` | The maximum frequency of operations (reads/writes).

**Applicable command:** `run`
**Default:** `0`, which means unlimited. +`--method` | The SQL issue method (`prepare`, `noprepare`, `simple`).

**Applicable commands:** `init` or `run`
**Default:** `prepare` +`--num-histories` | The initial number of ride location histories.

**Applicable commands:** `init` or `run`
**Default:** `1000` +`--num-promo-codes` | The initial number of promo codes.

**Applicable commands:** `init` or `run`
**Default:** `1000` +`--num-rides` | Initial number of rides.

**Applicable commands:** `init` or `run`
**Default:** `500` +`--num-users` | Initial number of users.

**Applicable commands:** `init` or `run`
**Default:** `50` +`--num-vehicles` | Initial number of vehicles.

**Applicable commands:** `init` or `run`
**Default:** `15` +`--ramp` | The duration over which to ramp up load.

**Applicable command:** `run` +`--seed` | The random number generator seed.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--tolerate-errors` | Keep running on error.

**Applicable command:** `run` + +### `tpcc` workload + +Flag | Description +-----|------------ +`--active-warehouses` | Run the load generator against a specific number of warehouses.

**Applicable commands:** `init` or `run`
**Defaults:** Value of `--warehouses` +`--db` | The SQL database to use.

**Applicable commands:** `init` or `run`
**Default:** `tpcc` +`--display-every` | The frequency for printing per-operation statistics. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1s` +`--display-format` | The format for printing per-operation statistics (`simple`, `incremental-json`). When using `incremental-json`, note that totals are not printed at the end of the workload's duration.

**Applicable command:** `run`
**Default:** `simple` +`--drop` | Drop the existing database, if it exists.

**Applicable commands:** `init` or `run`. For the `run` command, this flag must be used in conjunction with `--init`. +`--duration` | The duration to run, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `0`, which means run forever. +`--fks` | Add foreign keys.

**Applicable commands:** `init` or `run`
**Default:** `true` +`--histograms` | The file to write per-op incremental and cumulative histogram data to.

**Applicable command:** `run` +`--init` | **Deprecated.** Use the `init` command instead.

**Applicable command:** `run` +`--interleaved` | Use [interleaved tables](interleave-in-parent.html).

**Applicable commands:** `init` or `run` +`--max-ops` | The maximum number of operations to run.

**Applicable command:** `run` +`--max-rate` | The maximum frequency of operations (reads/writes).

**Applicable command:** `run`
**Default:** `0`, which means unlimited. +`--mix` | Weights for the transaction mix.

**Applicable commands:** `init` or `run`
**Default:** `newOrder=10,payment=10,orderStatus=1,delivery=1,stockLevel=1`, which matches the [TPC-C specification](http://www.tpc.org/tpc_documents_current_versions/current_specifications.asp). +`--partition-affinity` | Run the load generator against a specific partition. This flag must be used in conjunction with `--partitions`.

**Applicable commands:** `init` or `run`
**Default:** `-1` +`--partitions` | Partition tables. This flag must be used in conjunction with `--split`.

**Applicable commands:** `init` or `run` +`--ramp` | The duration over which to ramp up load.

**Applicable command:** `run` +`--scatter` | Scatter ranges.

**Applicable commands:** `init` or `run` +`--seed` | The random number generator seed.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--serializable` | Force serializable mode. CockroachDB only supports `SERIALIZABLE` isolation, so this flag is not necessary.

**Applicable command:** `init` +`--split` | [Split tables](split-at.html).

**Applicable commands:** `init` or `run` +`--tolerate-errors` | Keep running on error.

**Applicable command:** `run` +`--wait` | Run in wait mode, i.e., include think/keying sleeps.

**Applicable commands:** `init` or `run`
**Default:** `true` +`--warehouses` | The number of warehouses for loading initial data, at approximately 200 MB per warehouse.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--workers` | The number of concurrent workers.

**Applicable commands:** `init` or `run`
**Default:** `--warehouses` * 10 +`--zones` | The number of [replication zones](configure-replication-zones.html) for partitioning. This number should match the number of `--partitions` and the zones used to start the cluster.

**Applicable command:** `init` + +### `ycsb` workload + +Flag | Description +-----|------------ +`--concurrency` | The number of concurrent workers.

**Applicable commands:** `init` or `run`
**Default:** `8` +`--db` | The SQL database to use.

**Applicable commands:** `init` or `run`
**Default:** `ycsb` +`--display-every` | The frequency for printing per-operation statistics. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `1s` +`--display-format` | The format for printing per-operation statistics (`simple`, `incremental-json`). When using `incremental-json`, note that totals are not printed at the end of the workload's duration.

**Applicable command:** `run`
**Default:** `simple` +`--drop` | Drop the existing database, if it exists.

**Applicable commands:** `init` or `run`. For the `run` command, this flag must be used in conjunction with `--init`. +`--duration` | The duration to run, with a required time unit suffix. Valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)) are `ns`, `us`, `ms`, `s`, `m`, and `h`.

**Applicable command:** `run`
**Default:** `0`, which means run forever. +`--families` | Place each column in its own [column family](column-families.html).

**Applicable commands:** `init` or `run` +`--histograms` | The file to write per-op incremental and cumulative histogram data to.

**Applicable command:** `run` +`--init` | **Deprecated.** Use the `init` command instead.

**Applicable command:** `run` +`--initial-count` | Initial number of rows to sequentially insert before beginning random number generation.

**Applicable commands:** `init` or `run`
**Default:** `10000` +`--json` | Use JSONB rather than relational data.

**Applicable commands:** `init` or `run` +`--max-ops` | The maximum number of operations to run.

**Applicable command:** `run` +`--max-rate` | The maximum frequency of operations (reads/writes).

**Applicable command:** `run`
**Default:** `0`, which means unlimited. +`--method` | The SQL issue method (`prepare`, `noprepare`, `simple`).

**Applicable commands:** `init` or `run`
**Default:** `prepare` +`--ramp` | The duration over which to ramp up load.

**Applicable command:** `run` +`--request-distribution` | Distribution for the random number generator (`zipfian`, `uniform`).

**Applicable commands:** `init` or `run`.
**Default:** `zipfian` +`--seed` | The random number generator seed.

**Applicable commands:** `init` or `run`
**Default:** `1` +`--splits` | Number of [splits](split-at.html) to perform before starting normal operations.

**Applicable commands:** `init` or `run` +`--tolerate-errors` | Keep running on error.

**Applicable command:** `run` +`--workload` | The type of workload to run (`A`, `B`, `C`, `D`, or `F`). For details about these workloads, see [YCSB Workloads](https://github.com/brianfrankcooper/YCSB/wiki/Core-Workloads).

**Applicable commands:** `init` or `run`
**Default:** `B` + +### Logging + +By default, the `cockroach workload` command logs errors to `stderr`. + +If you need to troubleshoot this command's behavior, you can change its [logging behavior](debug-and-error-logs.html). + +## Examples + +These examples assume that you have already [started an insecure cluster locally](start-a-local-cluster.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--listen-addr=localhost +~~~ + +### Run the `bank` workload + +1. Load the initial schema: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init bank \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Run the workload for 1 minute: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run bank \ + --duration=1m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1s 0 1608.6 1702.2 4.5 7.3 12.6 65.0 transfer + 2s 0 1725.3 1713.8 4.5 7.9 13.1 19.9 transfer + 3s 0 1721.1 1716.2 4.5 7.3 11.5 21.0 transfer + 4s 0 1328.7 1619.2 5.5 10.5 17.8 39.8 transfer + 5s 0 1389.3 1573.3 5.2 11.5 16.3 23.1 transfer + 6s 0 1640.0 1584.4 5.0 7.9 12.1 16.3 transfer + 7s 0 1594.0 1585.8 5.0 7.9 10.5 15.7 transfer + 8s 0 1652.8 1594.2 4.7 7.9 11.5 29.4 transfer + 9s 0 1451.9 1578.4 5.2 10.0 15.2 26.2 transfer + 10s 0 1653.3 1585.9 5.0 7.6 10.0 18.9 transfer + ... + ~~~ + + After the specified duration (1 minute in this case), the workload will stop and you'll see totals printed to standard output: + + ~~~ + _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result + 60.0s 0 84457 1407.6 5.7 5.5 10.0 15.2 167.8 + ~~~ + +### Run the `kv` workload + +1. Load the initial schema: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init kv \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Run the workload for 1 minute: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run kv \ + --duration=1m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1s 0 5095.8 5123.7 1.5 2.5 3.3 7.3 write + 2s 0 4795.4 4959.6 1.6 2.8 3.5 8.9 write + 3s 0 3456.5 4458.5 2.0 4.5 7.3 24.1 write + 4s 0 2787.9 4040.8 2.4 6.3 12.6 30.4 write + 5s 0 3558.7 3944.4 2.0 4.2 6.8 11.5 write + 6s 0 3733.8 3909.3 1.9 4.2 6.0 12.6 write + 7s 0 3565.6 3860.1 2.0 4.7 7.9 25.2 write + 8s 0 3469.3 3811.4 2.0 5.0 6.8 22.0 write + 9s 0 3937.6 3825.4 1.8 3.7 7.3 29.4 write + 10s 0 3822.9 3825.1 1.8 4.7 8.9 37.7 write + ... + ~~~ + + After the specified duration (1 minute in this case), the workload will stop and you'll see totals printed to standard output: + + ~~~ + _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result + 60.0s 0 276067 4601.0 1.7 1.6 3.1 5.2 96.5 + ~~~ + +### Load the `intro` dataset + +1. Load the dataset: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init intro \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Launch the built-in SQL client to view it: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW TABLES FROM intro; + ~~~ + + ~~~ + table_name + +------------+ + mytable + (1 row) + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ SELECT * FROM intro.mytable WHERE (l % 2) = 0; + ~~~ + + ~~~ + l | v + +----+------------------------------------------------------+ + 0 | !__aaawwmqmqmwwwaas,,_ .__aaawwwmqmqmwwaaa,, + 2 | !"VT?!"""^~~^"""??T$Wmqaa,_auqmWBT?!"""^~~^^""??YV^ + 4 | ! "?##mW##?"- + 6 | ! C O N G R A T S _am#Z??A#ma, Y + 8 | ! _ummY" "9#ma, A + 10 | ! vm#Z( )Xmms Y + 12 | ! .j####mmm#####mm#m##6. + 14 | ! W O W ! jmm###mm######m#mmm##6 + 16 | ! ]#me*Xm#m#mm##m#m##SX##c + 18 | ! dm#||+*$##m#mm#m#Svvn##m + 20 | ! :mmE=|+||S##m##m#1nvnnX##; A + 22 | ! :m#h+|+++=Xmm#m#1nvnnvdmm; M + 24 | ! Y $#m>+|+|||##m#1nvnnnnmm# A + 26 | ! O ]##z+|+|+|3#mEnnnnvnd##f Z + 28 | ! U D 4##c|+|+|]m#kvnvnno##P E + 30 | ! I 4#ma+|++]mmhvnnvq##P` ! + 32 | ! D I ?$#q%+|dmmmvnnm##! + 34 | ! T -4##wu#mm#pw##7' + 36 | ! -?$##m####Y' + 38 | ! !! "Y##Y"- + 40 | ! + (21 rows) + ~~~ + +### Load the `startrek` dataset + +1. Load the dataset: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init startrek \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Launch the built-in SQL client to view it: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW TABLES FROM startrek; + ~~~ + + ~~~ + table_name + +------------+ + episodes + quotes + (2 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM startrek.episodes WHERE stardate > 5500; + ~~~ + + ~~~ + id | season | num | title | stardate + +----+--------+-----+-----------------------------------+----------+ + 60 | 3 | 5 | Is There in Truth No Beauty? | 5630.7 + 62 | 3 | 7 | Day of the Dove | 5630.3 + 64 | 3 | 9 | The Tholian Web | 5693.2 + 65 | 3 | 10 | Plato's Stepchildren | 5784.2 + 66 | 3 | 11 | Wink of an Eye | 5710.5 + 69 | 3 | 14 | Whom Gods Destroy | 5718.3 + 70 | 3 | 15 | Let That Be Your Last Battlefield | 5730.2 + 73 | 3 | 18 | The Lights of Zetar | 5725.3 + 74 | 3 | 19 | Requiem for Methuselah | 5843.7 + 75 | 3 | 20 | The Way to Eden | 5832.3 + 76 | 3 | 21 | The Cloud Minders | 5818.4 + 77 | 3 | 22 | The Savage Curtain | 5906.4 + 78 | 3 | 23 | All Our Yesterdays | 5943.7 + 79 | 3 | 24 | Turnabout Intruder | 5928.5 + (14 rows) + ~~~ + +### Load the `movr` dataset + +1. Load the dataset: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init movr \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Launch the built-in SQL client to view it: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW TABLES FROM movr; + ~~~ + + ~~~ + table_name ++----------------------------+ + promo_codes + rides + user_promo_codes + users + vehicle_location_histories + vehicles +(6 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM movr.users WHERE city='new york'; + ~~~ + + ~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 +(5 rows) + ~~~ + +### Run the `movr` workload + +1. Load the initial schema: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init movr \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Initialize and run the workload for 1 minute: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run movr \ + --duration=1m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1.0s 0 31.9 32.0 0.5 0.6 1.4 1.4 addUser + 1.0s 0 6.0 6.0 1.2 1.4 1.4 1.4 addVehicle + 1.0s 0 10.0 10.0 2.2 6.3 6.3 6.3 applyPromoCode + 1.0s 0 2.0 2.0 0.5 0.6 0.6 0.6 createPromoCode + 1.0s 0 9.0 9.0 0.9 1.6 1.6 1.6 endRide + 1.0s 0 1407.5 1407.8 0.3 0.5 0.7 4.1 readVehicles + 1.0s 0 27.0 27.0 2.1 3.1 4.7 4.7 startRide + 1.0s 0 86.8 86.9 4.7 8.4 11.5 15.2 updateActiveRides + 2.0s 0 26.0 29.0 0.5 1.1 1.4 1.4 addUser + 2.0s 0 8.0 7.0 1.2 2.8 2.8 2.8 addVehicle + 2.0s 0 2.0 6.0 2.6 2.8 2.8 2.8 applyPromoCode + 2.0s 0 0.0 1.0 0.0 0.0 0.0 0.0 createPromoCode + 2.0s 0 6.0 7.5 0.8 1.7 1.7 1.7 endRide + 2.0s 0 1450.4 1429.1 0.3 0.6 0.9 2.6 readVehicles + 2.0s 0 17.0 22.0 2.1 3.3 5.5 5.5 startRide + 2.0s 0 59.0 72.9 6.3 11.5 11.5 14.2 updateActiveRides + ... + ~~~ + + After the specified duration (1 minute in this case), the workload will stop and you'll see totals printed to standard output: + + ~~~ + _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result + 60.0s 0 85297 1421.6 0.7 0.3 2.6 7.1 30.4 + ~~~ + +### Run the `tpcc` workload + +1. Load the initial schema and data: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init tpcc \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Run the workload for 10 minutes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run tpcc \ + --duration=10m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1s 0 1443.4 1494.8 4.7 9.4 27.3 67.1 transfer + 2s 0 1686.5 1590.9 4.7 8.1 15.2 28.3 transfer + 3s 0 1735.7 1639.0 4.7 7.3 11.5 28.3 transfer + 4s 0 1542.6 1614.9 5.0 8.9 12.1 21.0 transfer + 5s 0 1695.9 1631.1 4.7 7.3 11.5 22.0 transfer + 6s 0 1569.2 1620.8 5.0 8.4 11.5 15.7 transfer + 7s 0 1614.6 1619.9 4.7 8.1 12.1 16.8 transfer + 8s 0 1344.4 1585.6 5.8 10.0 15.2 31.5 transfer + 9s 0 1351.9 1559.5 5.8 10.0 16.8 54.5 transfer + 10s 0 1514.8 1555.0 5.2 8.1 12.1 16.8 transfer + ... + ~~~ + + After the specified duration (10 minutes in this case), the workload will stop and you'll see totals printed to standard output: + + ~~~ + _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result + 600.0s 0 823902 1373.2 5.8 5.5 10.0 15.2 209.7 + ~~~ + +### Run the `ycsb` workload + +1. Load the initial schema and data: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init ycsb \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Run the workload for 10 minutes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run ycsb \ + --duration=10m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1s 0 9258.1 9666.6 0.7 1.3 2.0 8.9 read + 1s 0 470.1 490.9 1.7 2.9 4.1 5.0 update + 2s 0 10244.6 9955.6 0.7 1.2 2.0 6.6 read + 2s 0 559.0 525.0 1.6 3.1 6.0 7.3 update + 3s 0 9870.8 9927.4 0.7 1.4 2.4 10.0 read + 3s 0 500.0 516.6 1.6 4.2 7.9 15.2 update + 4s 0 9847.2 9907.3 0.7 1.4 2.4 23.1 read + 4s 0 506.8 514.2 1.6 3.7 7.6 17.8 update + 5s 0 10084.4 9942.6 0.7 1.3 2.1 7.1 read + 5s 0 537.2 518.8 1.5 3.5 10.0 15.2 update + ... + ~~~ + + After the specified duration (10 minutes in this case), the workload will stop and you'll see totals printed to standard output: + + ~~~ + _elapsed___errors_____ops(total)___ops/sec(cum)__avg(ms)__p50(ms)__p95(ms)__p99(ms)_pMax(ms)__result + 600.0s 0 4728286 7880.2 1.0 0.9 2.2 5.2 268.4 + ~~~ + +### Customize the frequency and format of per-operation statistics + +To customize the frequency of per-operation statistics, use the `--display-every` flag, with `ns`, `us`, `ms`, `s`, `m`, and `h` as valid [time units](https://en.wikipedia.org/wiki/Orders_of_magnitude_(time)). To customize the format of per-operation statistics, use the `--display-format` flag, with `incremental-json` or `simple` (default) as options. + +1. Load the initial schema and data: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init ycsb \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Run the workload for 1 minute, printing the output every 5 seconds as JSON: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run ycsb \ + --duration=1m \ + --display-every=5s \ + --display-format=incremental-json \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + ~~~ + {"time":"2019-09-13T03:25:03.950621Z","errs":0,"avgt":8434.5,"avgl":8471.0,"p50l":0.8,"p95l":1.6,"p99l":3.1,"maxl":19.9,"type":"read"} + {"time":"2019-09-13T03:25:03.950621Z","errs":0,"avgt":438.1,"avgl":440.0,"p50l":1.5,"p95l":2.8,"p99l":4.5,"maxl":14.7,"type":"update"} + {"time":"2019-09-13T03:25:08.95061Z","errs":0,"avgt":7610.6,"avgl":8040.8,"p50l":0.8,"p95l":2.0,"p99l":4.2,"maxl":65.0,"type":"read"} + {"time":"2019-09-13T03:25:08.95061Z","errs":0,"avgt":391.8,"avgl":415.9,"p50l":1.6,"p95l":3.5,"p99l":5.8,"maxl":21.0,"type":"update"} + {"time":"2019-09-13T03:25:13.950727Z","errs":0,"avgt":7242.0,"avgl":7774.5,"p50l":0.8,"p95l":2.2,"p99l":4.7,"maxl":75.5,"type":"read"} + {"time":"2019-09-13T03:25:13.950727Z","errs":0,"avgt":382.0,"avgl":404.6,"p50l":1.6,"p95l":4.7,"p99l":10.5,"maxl":24.1,"type":"update"} + ... + ~~~ + + When using `incremental-json`, note that totals are not printed at the end of the workload's duration. + +## See also + +- [`cockroach demo`](cockroach-demo.html) +- [Other Cockroach Commands](cockroach-commands.html) +- [Performance Benchmarking with TPC-C](performance-benchmarking-with-tpc-c-1k-warehouses.html) diff --git a/v20.2/cockroachdb-architecture.md b/v20.2/cockroachdb-architecture.md new file mode 100644 index 00000000000..ff03d174a68 --- /dev/null +++ b/v20.2/cockroachdb-architecture.md @@ -0,0 +1,3 @@ +--- +redirect_to: architecture/overview.html +--- diff --git a/v20.2/cockroachdb-in-comparison.md b/v20.2/cockroachdb-in-comparison.md new file mode 100644 index 00000000000..25524e4a635 --- /dev/null +++ b/v20.2/cockroachdb-in-comparison.md @@ -0,0 +1,351 @@ +--- +title: CockroachDB in Comparison +summary: Learn how CockroachDB compares to other popular databases like PostgreSQL, Cassandra, MongoDB, Google Cloud Spanner, and more. +tags: mongodb, mysql, dynamodb +toc: false +comparison: true +--- + +This page shows you how key features of CockroachDB stack up against other databases. Hover over features for their intended meanings, and click CockroachDB answers to view related documentation. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + CockroachDB
+ Database Horizontal Scale + + tooltip icon + + + Manual Sharding + Add On Configuration + Node based, Automated read scale, limited write + Node based, Automated for both reads and writes + + Manual Sharding + Add On Configuration + Node based, Automated read scale, limited write + Node based, Automated for both reads and writes + Node based, Automated for both reads and writes
+ Database Load Balancing (internal) + + tooltip icon + + + Manual - not part of database + None and Full copies across regions + Even distribution to optimize storage + + Manual - not part of database + None and Full copies across regions + Even distribution to optimize storage + Detailed options to optimize storage, compute and latency
+ Failover + + tooltip icon + + + Manual - not part of database + Automated for reads, limited for writes to one region + Automated for reads, limited guarantees for writes + Fully automated for both reads and writes + + Manual - not part of database + Automated for reads, limited for writes to one region + Automated for reads, limited guarantees for writes + Fully automated for both reads and writes + Fully automated for both reads and writes
+ Automated Repair and RPO + + tooltip icon + + + Manual Repair RPO ~1-60 mins + Automated RPO ~1 -5 mins + Manual & Automated Repair RPO <1 min + "Automated Repair RPO <10 sec" + + Manual Repair RPO ~1-60 mins + Automated RPO ~1 -5 mins + Manual & Automated Repair RPO <1 min + "Automated Repair RPO <10 sec" + Automated Repair RPO <10 sec
+ Distributed Reads + + tooltip icon + + + Manual - Asynchronous + Yes + + Manual - Asynchronous + Yes + Yes
+ Distributed Transactions + + tooltip icon + + + No + Lightweight Transactions only + Yes + + No + Lightweight Transactions only + Yes + Yes
+ Database Isolation Levels + + tooltip icon + + + Single Region Consistent Default: Snapshot Highest: Serializable + Eventual Consistent Default: Read Uncommited Highest: Snapshot Read + Eventual Consistent, No Transaction Isolation Guarantees + Default: Snapshot Highest: Serializable + + Single Region Consistent Default: Snapshot Highest: Serializable + Eventual Consistent Default: Read Uncommited Highest: Snapshot Read + Eventual Consistent, No Transaction Isolation Guarantees + Default: Snapshot Highest: Serializable + Guaranteed Consistent Default: Serializable Highest: Serializable
+ Potential data issues (default) + + tooltip icon + + + Phantom Reads, Non-repeatable reads, Write skew + Dirty Reads, Phantom Reads, Non-repeatable reads, write skew + Dirty Reads, Phantom Reads, Non-repeatable reads, write conflicts + None + Phantom Reads, Non-repeatable reads + + Phantom Reads, Non-repeatable reads, Write skew + Dirty Reads, Phantom Reads, Non-repeatable reads, write skew + Dirty Reads, Phantom Reads, Non-repeatable reads, write conflicts + None + Phantom Reads, Non-repeatable reads + None
+ SQL + + tooltip icon + + + Yes + No + Yes - with limitations + + Yes + No + Yes - with limitations + Yes - wire compatible with PostgreSQL
+ Database Schema Change + + tooltip icon + + + Yes + Offline + Online, Active, Dynamic + + Yes + Offline + Online, Active, Dynamic + Online, Active, Dynamic
+ Cost Based Optimization + + tooltip icon + + + Yes + No + ? + No + + Yes + No + ? + No + Yes
+ Data Geo-partitoning + + tooltip icon + + + No + Yes, Object Level + Yes + No + + No + Yes, Object Level + Yes + No + Yes, Row level
+ Upgrade Method + + tooltip icon + + + Offline + Online, Rolling + + Offline + Online, Rolling + Online, Rolling
+ Multi-region + + tooltip icon + + + Yes - Manual + Yes, but not for writes + Yes, for reads and writes + + Yes - Manual + Yes, but not for writes + Yes, for reads and writes + Yes for both reads and writes
+ Multi-cloud + + tooltip icon + + + No + Yes + + No + Yes + Yes
+ + diff --git a/v20.2/collate.md b/v20.2/collate.md new file mode 100644 index 00000000000..2eca0a4a630 --- /dev/null +++ b/v20.2/collate.md @@ -0,0 +1,177 @@ +--- +title: COLLATE +summary: The COLLATE feature lets you sort strings according to language- and country-specific rules. +toc: true +redirect_from: collatedstring.html +--- + +The `COLLATE` feature lets you sort [`STRING`](string.html) values according to language- and country-specific rules, known as collations. + +Collated strings are important because different languages have [different rules for alphabetic order](https://en.wikipedia.org/wiki/Alphabetical_order#Language-specific_conventions), especially with respect to accented letters. For example, in German accented letters are sorted with their unaccented counterparts, while in Swedish they are placed at the end of the alphabet. A collation is a set of rules used for ordering and usually corresponds to a language, though some languages have multiple collations with different rules for sorting; for example Portuguese has separate collations for Brazilian and European dialects (`pt-BR` and `pt-PT` respectively). + +{% include {{page.version.version}}/sql/vectorized-support.md %} + +## Details + +- Operations on collated strings cannot involve strings with a different collation or strings with no collation. However, it is possible to add or overwrite a collation on the fly. + +- Only use the collation feature when you need to sort strings by a specific collation. We recommend this because every time a collated string is constructed or loaded into memory, CockroachDB computes its collation key, whose size is linear in relationship to the length of the collated string, which requires additional resources. + +- Collated strings can be considerably larger than the corresponding uncollated strings, depending on the language and the string content. For example, strings containing the character `é` produce larger collation keys in the French locale than in Chinese. + +- Collated strings that are indexed require additional disk space as compared to uncollated strings. In case of indexed collated strings, collation keys must be stored in addition to the strings from which they are derived, creating a constant factor overhead. + +## Supported collations + +CockroachDB supports the collations provided by Go's [language package](https://godoc.org/golang.org/x/text/language#Tag). The `` argument is the BCP 47 language tag at the end of each line, immediately preceded by `//`. For example, Afrikaans is supported as the `af` collation. + +## SQL syntax + +Collated strings are used as normal strings in SQL, but have a `COLLATE` clause appended to them. + +- **Column syntax**: `STRING COLLATE `. For example: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE foo (a STRING COLLATE en PRIMARY KEY); + ~~~ + + {{site.data.alerts.callout_info}}You can also use any of the aliases for STRING.{{site.data.alerts.end}} + +- **Value syntax**: ` COLLATE `. For example: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO foo VALUES ('dog' COLLATE en); + ~~~ + +## Examples + +### Specify collation for a column + +You can set a default collation for all values in a `STRING` column. + +For example, you can set a column's default collation to German (`de`): + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE de_names (name STRING COLLATE de PRIMARY KEY); +~~~ + +When inserting values into this column, you must specify the collation for every value: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO de_names VALUES ('Backhaus' COLLATE de), ('Bär' COLLATE de), ('Baz' COLLATE de); +~~~ + +The sort will now honor the `de` collation that treats *ä* as *a* in alphabetic sorting: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM de_names ORDER BY name; +~~~ +~~~ + name ++----------+ + Backhaus + Bär + Baz +(3 rows) +~~~ + +### Order by non-default collation + +You can sort a column using a specific collation instead of its default. + +For example, you receive different results if you order results by German (`de`) and Swedish (`sv`) collations: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM de_names ORDER BY name COLLATE sv; +~~~ +~~~ + name ++----------+ + Backhaus + Baz + Bär +(3 rows) +~~~ + +### Ad-hoc collation casting + +You can cast any string into a collation on the fly. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT 'A' COLLATE de < 'Ä' COLLATE de; +~~~ +~~~ + ?column? ++----------+ + true +(1 row) +~~~ + +However, you cannot compare values with different collations: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT 'Ä' COLLATE sv < 'Ä' COLLATE de; +~~~ +~~~ +pq: unsupported comparison operator: < +~~~ + +You can also use casting to remove collations from values. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT CAST(name AS STRING) FROM de_names ORDER BY name; +~~~ +~~~ + name ++----------+ + Backhaus + Baz + Bär +(3 rows) +~~~ + +### Show collation for strings + +You can use the `pg_collation_for` [built-in function](functions-and-operators.html#string-and-byte-functions), or its alternative [syntax form](functions-and-operators.html#special-syntax-forms) `COLLATION FOR`, to return the locale name of a collated string. + +For example: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT pg_collation_for('Bär' COLLATE de); +~~~ + +~~~ + pg_collation_for ++------------------+ + de +(1 row) +~~~ + +This is equivalent to: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT COLLATION FOR ('Bär' COLLATE de); +~~~ + +~~~ + pg_collation_for ++------------------+ + de +(1 row) +~~~ + + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/column-families.md b/v20.2/column-families.md new file mode 100644 index 00000000000..6755913b178 --- /dev/null +++ b/v20.2/column-families.md @@ -0,0 +1,94 @@ +--- +title: Column Families +summary: A column family is a group of columns in a table that are stored as a single key-value pair in the underlying key-value store. +toc: true +--- + +A column family is a group of columns in a table that are stored as a single key-value pair in the [underlying key-value store](architecture/storage-layer.html). Column families reduce the number of keys stored in the key-value store, resulting in improved performance during [`INSERT`](insert.html), [`UPDATE`](update.html), and [`DELETE`](delete.html) operations. + +This page explains how CockroachDB organizes columns into families as well as cases in which you might want to manually override the default behavior. + + [Secondary indexes](indexes.html) respect the column family definitions applied to tables. When you define a secondary index, CockroachDB breaks the secondary index key-value pairs into column families, according to the family and stored column configurations. + +## Default behavior + +When a table is created, all columns are stored as a single column family. + +This default approach ensures efficient key-value storage and performance in most cases. However, when frequently updated columns are grouped with seldom updated columns, the seldom updated columns are nonetheless rewritten on every update. Especially when the seldom updated columns are large, it's more performant to split them into a distinct family. + +## Manual override + +### Assign column families on table creation + +To manually assign a column family on [table creation](create-table.html), use the `FAMILY` keyword. + +For example, let's say we want to create a table to store an immutable blob of data (`data BYTES`) with a last accessed timestamp (`last_accessed TIMESTAMP`). Because we know that the blob of data will never get updated, we use the `FAMILY` keyword to break it into a separate column family: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE test ( + id INT PRIMARY KEY, + last_accessed TIMESTAMP, + data BYTES, + FAMILY f1 (id, last_accessed), + FAMILY f2 (data) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE test; +~~~ + +~~~ + table_name | create_statement +-------------+------------------------------------------------- + test | CREATE TABLE test ( + | id INT8 NOT NULL, + | last_accessed TIMESTAMP NULL, + | data BYTES NULL, + | CONSTRAINT "primary" PRIMARY KEY (id ASC), + | FAMILY f1 (id, last_accessed), + | FAMILY f2 (data) + | ) +(1 row) +~~~ + + +### Assign column families when adding columns + +When using the [`ALTER TABLE .. ADD COLUMN`](add-column.html) statement to add a column to a table, you can assign the column to a new or existing column family. + +- Use the `CREATE FAMILY` keyword to assign a new column to a **new family**. For example, the following would add a `data2 BYTES` column to the `test` table above and assign it to a new column family: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE test ADD COLUMN data2 BYTES CREATE FAMILY f3; + ~~~ + +- Use the `FAMILY` keyword to assign a new column to an **existing family**. For example, the following would add a `name STRING` column to the `test` table above and assign it to family `f1`: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE test ADD COLUMN name STRING FAMILY f1; + ~~~ + +- Use the `CREATE IF NOT EXISTS FAMILY` keyword to assign a new column to an **existing family or, if the family doesn't exist, to a new family**. For example, the following would assign the new column to the existing `f1` family; if that family didn't exist, it would create a new family and assign the column to it: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE test ADD COLUMN name STRING CREATE IF NOT EXISTS FAMILY f1; + ~~~ + +- If a column is added to a table and the family is not specified, it will be added to the first column family. For example, the following would add the new column to the `f1` family, since that is the first column family: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE test ADD COLUMN last_name STRING; + ~~~ + +## See also + +- [`CREATE TABLE`](create-table.html) +- [`ADD COLUMN`](add-column.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/comment-on.md b/v20.2/comment-on.md new file mode 100644 index 00000000000..39614019c89 --- /dev/null +++ b/v20.2/comment-on.md @@ -0,0 +1,185 @@ +--- +title: COMMENT ON +summary: The COMMENT ON statement associates comments to databases, tables, columns, or indexes. +toc: true +--- + +The `COMMENT ON` [statement](sql-statements.html) associates comments to [databases](create-database.html), [tables](create-table.html), [columns](add-column.html), or [indexes](indexes.html). + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the object they are commenting on. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/comment.html %}
+ +## Parameters + + Parameter | Description +------------|-------------- +`database_name` | The name of the database you are commenting on. +`table_name` | The name of the table you are commenting on. +`column_name` | The name of the column you are commenting on. +`table_index_name` | The name of the index you are commenting on. +`comment_text` | The comment ([`STRING`](string.html)) you are associating to the object. + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Add a comment to a database + +To add a comment to a database: + +{% include copy-clipboard.html %} +~~~ sql +> COMMENT ON DATABASE movr IS 'This database contains information about users, vehicles, and rides.'; +~~~ + +To view database comments, use [`SHOW DATABASES`](show-databases.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW DATABASES WITH COMMENT; +~~~ + +~~~ + database_name | comment ++---------------+-------------------------------------------------------------------+ + defaultdb | NULL + movr | This database contains information about users, vehicles, and rides. + postgres | NULL + system | NULL +(4 rows) +~~~ + +### Add a comment to a table + +To add a comment to a table: + +{% include copy-clipboard.html %} +~~~ sql +> COMMENT ON TABLE vehicles IS 'This table contains information about vehicles registered with MovR.'; +~~~ + +To view table comments, use [`SHOW TABLES`](show-tables.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM movr WITH COMMENT; +~~~ + +~~~ + table_name | comment ++----------------------------+----------------------------------------------------------------------+ + users | + vehicles | This table contains information about vehicles registered with MovR. + rides | + vehicle_location_histories | + promo_codes | + user_promo_codes | +(6 rows) +~~~ + + You can also view comments on a table with [`SHOW CREATE`](show-create.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE vehicles; +~~~ + +~~~ + table_name | create_statement +-------------+------------------------------------------------------------------------------------------------------ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | type VARCHAR NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status VARCHAR NULL, + | current_location VARCHAR NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | CONSTRAINT fk_city_ref_users FOREIGN KEY (city, owner_id) REFERENCES users(city, id), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ); + | COMMENT ON TABLE vehicles IS 'This table contains information about vehicles registered with MovR.' +(1 row) +~~~ + +### Add a comment to a column + +To add a comment to a column: + +{% include copy-clipboard.html %} +~~~ sql +> COMMENT ON COLUMN users.credit_card IS 'This column contains user payment information.'; +~~~ + +To view column comments, use [`SHOW COLUMNS`](show-columns.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM users WITH COMMENT; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden | comment ++-------------+-----------+-------------+----------------+-----------------------+-----------+-----------+------------------------------------------------+ + id | UUID | false | NULL | | {primary} | false | NULL + city | VARCHAR | false | NULL | | {primary} | false | NULL + name | VARCHAR | true | NULL | | {} | false | NULL + address | VARCHAR | true | NULL | | {} | false | NULL + credit_card | VARCHAR | true | NULL | | {} | false | This column contains user payment information. +(5 rows) +~~~ + +### Add a comment to an index + +Suppose we [create an index](create-index.html) on the `name` column of the `users` table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users(name); +~~~ + +To add a comment to the index: + +{% include copy-clipboard.html %} +~~~ sql +> COMMENT ON INDEX users_name_idx IS 'This index improves performance on queries that filter by name.'; +~~~ + +To view column comments, use [`SHOW INDEXES ... WITH COMMENT`](show-index.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEXES FROM users WITH COMMENT; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit | comment +-------------+----------------+------------+--------------+-------------+-----------+---------+----------+------------------------------------------------------------------ + users | primary | false | 1 | city | ASC | false | false | NULL + users | primary | false | 2 | id | ASC | false | false | NULL + users | users_name_idx | true | 1 | name | ASC | false | false | This index improves performance on queries that filter by name. + users | users_name_idx | true | 2 | city | ASC | false | true | This index improves performance on queries that filter by name. + users | users_name_idx | true | 3 | id | ASC | false | true | This index improves performance on queries that filter by name. + users | primary | false | 1 | city | ASC | false | false | NULL + users | primary | false | 2 | id | ASC | false | false | NULL +... +(15 rows) +~~~ + +## See also + +- [`CREATE DATABASE`](create-database.html) +- [`CREATE TABLE`](create-table.html) +- [`ADD COLUMN`](add-column.html) +- [`CREATE INDEX`](create-index.html) +- [`SHOW TABLES`](show-tables.html) +- [Other SQL Statements](sql-statements.html) +- [dBeaver](dbeaver.html) diff --git a/v20.2/commit-transaction.md b/v20.2/commit-transaction.md new file mode 100644 index 00000000000..79393f0cc95 --- /dev/null +++ b/v20.2/commit-transaction.md @@ -0,0 +1,84 @@ +--- +title: COMMIT +summary: Commit a transaction with the COMMIT statement in CockroachDB. +toc: true +--- + +The `COMMIT` [statement](sql-statements.html) commits the current [transaction](transactions.html) or, when using [advanced client-side transaction retries](advanced-client-side-transaction-retries.html), clears the connection to allow new transactions to begin. + +When using [advanced client-side transaction retries](advanced-client-side-transaction-retries.html), statements issued after [`SAVEPOINT`](savepoint.html) are committed when [`RELEASE SAVEPOINT`](release-savepoint.html) is issued instead of `COMMIT`. However, you must still issue a `COMMIT` statement to clear the connection for the next transaction. + +For non-retryable transactions, if statements in the transaction [generated any errors](transactions.html#error-handling), `COMMIT` is equivalent to `ROLLBACK`, which aborts the transaction and discards *all* updates made by its statements. + + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/commit_transaction.html %}
+ +## Required privileges + +No [privileges](authorization.html#assign-privileges) are required to commit a transaction. However, privileges are required for each statement within a transaction. + +## Aliases + +In CockroachDB, `END` is an alias for the `COMMIT` statement. + +## Example + +### Commit a transaction + +How you commit transactions depends on how your application handles [transaction retries](transactions.html#transaction-retries). + +#### Client-side retryable transactions + +When using [advanced client-side transaction retries](advanced-client-side-transaction-retries.html), statements are committed by [`RELEASE SAVEPOINT`](release-savepoint.html). `COMMIT` itself only clears the connection for the next transaction. + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE products SET inventory = 0 WHERE sku = '8675309'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders (customer, sku, status) VALUES (1001, '8675309', 'new'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> RELEASE SAVEPOINT cockroach_restart; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> COMMIT; +~~~ + +{{site.data.alerts.callout_danger}}This example assumes you're using client-side intervention to handle transaction retries.{{site.data.alerts.end}} + +#### Automatically retried transactions + +If you are using transactions that CockroachDB will [automatically retry](transactions.html#automatic-retries) (i.e., all statements sent in a single batch), commit the transaction with `COMMIT`. + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; UPDATE products SET inventory = 100 WHERE = '8675309'; UPDATE products SET inventory = 100 WHERE = '8675310'; COMMIT; +~~~ + +## See also + +- [Transactions](transactions.html) +- [`BEGIN`](begin-transaction.html) +- [`RELEASE SAVEPOINT`](release-savepoint.html) +- [`ROLLBACK`](rollback-transaction.html) +- [`SAVEPOINT`](savepoint.html) +- [`SHOW SAVEPOINT STATUS`](show-savepoint-status.html) diff --git a/v20.2/common-errors.md b/v20.2/common-errors.md new file mode 100644 index 00000000000..e260f6c2db2 --- /dev/null +++ b/v20.2/common-errors.md @@ -0,0 +1,232 @@ +--- +title: Common Errors +summary: Understand and resolve common errors. +toc: false +redirect_from: general-troubleshooting.html +--- + +This page helps you understand and resolve error messages written to `stderr` or your [logs](debug-and-error-logs.html). + +| Topic | Message | +|----------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Client connection | [`connection refused`](#connection-refused) | +| Client connection | [`node is running secure mode, SSL connection required`](#node-is-running-secure-mode-ssl-connection-required) | +| Transaction retries | [`restart transaction`](#restart-transaction) | +| Node startup | [`node belongs to cluster but is attempting to connect to a gossip network for cluster `](#node-belongs-to-cluster-cluster-id-but-is-attempting-to-connect-to-a-gossip-network-for-cluster-another-cluster-id) | +| Node configuration | [`clock synchronization error: this node is more than 500ms away from at least half of the known nodes`](#clock-synchronization-error-this-node-is-more-than-500ms-away-from-at-least-half-of-the-known-nodes) | +| Node configuration | [`open file descriptor limit of is under the minimum required `](#open-file-descriptor-limit-of-number-is-under-the-minimum-required-number) | +| Replication | [`replicas failing with "0 of 1 store with an attribute matching []; likely not enough nodes in cluster"`](#replicas-failing-with-0-of-1-store-with-an-attribute-matching-likely-not-enough-nodes-in-cluster) | +| Split failed | [`split failed while applying backpressure; are rows updated in a tight loop?`](#split-failed-while-applying-backpressure-are-rows-updated-in-a-tight-loop) | +| Deadline exceeded | [`context deadline exceeded`](#context-deadline-exceeded) | +| Ambiguous results | [`result is ambiguous`](#result-is-ambiguous) | + +## connection refused + +This message indicates a client is trying to connect to a node that is either not running or is not listening on the specified interfaces (i.e., hostname or port). + +To resolve this issue, do one of the following: + +- If the node hasn't yet been started, [start the node](cockroach-start.html). +- If you specified a [`--listen-addr` and/or a `--advertise-addr` flag](cockroach-start.html#networking) when starting the node, you must include the specified IP address/hostname and port with all other [`cockroach` commands](cockroach-commands.html) or change the `COCKROACH_HOST` environment variable. + +If you're not sure what the IP address/hostname and port values might have been, you can look in the node's [logs](debug-and-error-logs.html). If necessary, you can also kill the `cockroach` process, and then restart the node: + +{% include copy-clipboard.html %} +~~~ shell +$ pkill cockroach +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start [flags] +~~~ + +## node is running secure mode, SSL connection required + +This message indicates that the cluster is using TLS encryption to protect network communication, and the client is trying to open a connection without using the required TLS certificates. + +To resolve this issue, use the [`cockroach cert create-client`](cockroach-cert.html) command to generate a client certificate and key for the user trying to connect. For a secure deployment walkthrough, including generating security certificates and connecting clients, see [Manual Deployment](manual-deployment.html). + +## restart transaction + +Messages with the error code `40001` and the string `restart transaction` indicate that a transaction failed because it conflicted with another concurrent or recent transaction accessing the same data. The transaction needs to be retried by the client. See [client-side transaction retries](transactions.html#client-side-intervention) for more details. + +Several different types of transaction retry errors are described below: + +- [`read within uncertainty interval`](#read-within-uncertainty-interval) +- [`transaction deadline exceeded`](#transaction-deadline-exceeded) + +{{site.data.alerts.callout_info}} +Your application's retry logic does not need to distinguish between these types of errors. They are listed here for reference. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_success}} +To understand how transactions work in CockroachDB, and why transaction retries are necessary to maintain serializable isolation in a distributed database, see: + +- [Transaction Layer](architecture/transaction-layer.html) +- [Life of a Distributed Transaction](architecture/life-of-a-distributed-transaction.html) +{{site.data.alerts.end}} + +### read within uncertainty interval + +(Error string includes: `ReadWithinUncertaintyIntervalError`) + +Uncertainty errors can occur when two transactions which start on different gateway nodes attempt to operate on the same data at close to the same time. The uncertainty comes from the fact that we cannot tell which one started first - the clocks on the two gateway nodes may not be perfectly in sync. + +For example, if the clock on node A is ahead of the clock on node B, a transaction started on node A may be able to commit a write with a timestamp that is still in the "future" from the perspective of node B. A later transaction that starts on node B should be able to see the earlier write from node A, even if B's clock has not caught up to A. The "read within uncertainty interval" occurs if we discover this situation in the middle of a transaction, when it is too late for the database to handle it automatically. When node B's transaction retries, it will unambiguously occur after the transaction from node A. + +Note that as long as the [client-side retry protocol](transactions.html#client-side-intervention) is followed, a transaction that has restarted once is much less likely to hit another uncertainty error, and the [`--max-offset` option](cockroach-start.html#flags) provides an upper limit on how long a transaction can continue to restart due to uncertainty. + +When errors like this occur, the application has the following options: + +- Prefer consistent historical reads using [AS OF SYSTEM TIME](as-of-system-time.html) to reduce contention. +- Design the schema and queries to reduce contention. For information on how to avoid contention, see [Understanding and Avoiding Transaction Contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention). +- Be prepared to retry on uncertainty (and other) errors. For more information, see [Transaction retries](transactions.html#transaction-retries). + +{{site.data.alerts.callout_info}} +Uncertainty errors are a form of transaction conflict. For more information about transaction conflicts, see [Transaction conflicts](architecture/transaction-layer.html#transaction-conflicts). +{{site.data.alerts.end}} + +### transaction deadline exceeded + +Errors which were previously reported to the client as opaque `TransactionStatusError`s are now transaction retry errors with the error message "transaction deadline exceeded" and error code `40001`. + +This error can occur for long-running transactions (with execution time on the order of minutes) that also experience conflicts with other transactions and thus attempt to commit at a timestamp different than their original timestamp. If the timestamp at which the transaction attempts to commit is above a "deadline" imposed by the various schema elements that the transaction has used (i.e. table structures), then this error might get returned to the client. + +When this error occurs, the application must retry the transaction. For more information about how to retry transactions, see [Transaction retries](transactions.html#transaction-retries). + +{{site.data.alerts.callout_info}} +For more information about the mechanics of the transaction conflict resolution process described above, see [Life of a Distributed Transaction](architecture/life-of-a-distributed-transaction.html). +{{site.data.alerts.end}} + + + + + +## node belongs to cluster \ but is attempting to connect to a gossip network for cluster \ + +This message usually indicates that a node tried to connect to a cluster, but the node is already a member of a different cluster. This is determined by metadata in the node's data directory. To resolve this issue, do one of the following: + +- Choose a different directory to store the CockroachDB data: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start [flags] --store=[new directory] --join=[cluster host]:26257 + ~~~ + +- Remove the existing directory and start a node joining the cluster again: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm -r cockroach-data/ + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start [flags] --join=[cluster host]:26257 + ~~~ + +## clock synchronization error: this node is more than 500ms away from at least half of the known nodes + +This error indicates that a node has spontaneously shut down because it detected that its clock is out of sync with at least half of the other nodes in the cluster by 80% of the maximum offset allowed (500ms by default). CockroachDB requires moderate levels of [clock synchronization](recommended-production-settings.html#clock-synchronization) to preserve data consistency, so the node shutting down in this way avoids the risk of consistency anomalies. + +To prevent this from happening, you should run clock synchronization software on each node. For guidance on synchronizing clocks, see the tutorial for your deployment environment: + +Environment | Recommended Approach +------------|--------------------- +[Manual](deploy-cockroachdb-on-premises.html#step-1-synchronize-clocks) | Use NTP with Google's external NTP service. +[AWS](deploy-cockroachdb-on-aws.html#step-3-synchronize-clocks) | Use the Amazon Time Sync Service. +[Azure](deploy-cockroachdb-on-microsoft-azure.html#step-3-synchronize-clocks) | Disable Hyper-V time synchronization and use NTP with Google's external NTP service. +[Digital Ocean](deploy-cockroachdb-on-digital-ocean.html#step-2-synchronize-clocks) | Use NTP with Google's external NTP service. +[GCE](deploy-cockroachdb-on-google-cloud-platform.html#step-3-synchronize-clocks) | Use NTP with Google's internal NTP service. + +## open file descriptor limit of \ is under the minimum required \ + +CockroachDB can use a large number of open file descriptors, often more than is available by default. This message indicates that the machine on which a CockroachDB node is running is under CockroachDB's recommended limits. + +For more details on CockroachDB's file descriptor limits and instructions on increasing the limit on various platforms, see [File Descriptors Limit](recommended-production-settings.html#file-descriptors-limit). + +## replicas failing with "0 of 1 store with an attribute matching []; likely not enough nodes in cluster + +### When running a single-node cluster + +When running a single-node CockroachDB cluster, an error about replicas failing will eventually show up in the node's log files, for example: + +~~~ shell +E160407 09:53:50.337328 storage/queue.go:511 [replicate] 7 replicas failing with "0 of 1 store with an attribute matching []; likely not enough nodes in cluster" +~~~ + +This happens because CockroachDB expects three nodes by default. If you do not intend to add additional nodes, you can stop this error by using [`ALTER RANGE ... CONFIGURE ZONE`](configure-zone.html) to update your default zone configuration to expect only one node: + +{% include copy-clipboard.html %} +~~~ shell +# Insecure cluster: +$ cockroach sql --execute="ALTER RANGE default CONFIGURE ZONE USING num_replicas=1;" --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ shell +# Secure cluster: +$ cockroach sql --execute="ALTER RANGE default CONFIGURE ZONE USING num_replicas=1;" --certs-dir=[path to certs directory] +~~~ + +The zone's replica count is reduced to 1. For more information, see [`ALTER RANGE ... CONFIGURE ZONE`](configure-zone.html) and [Configure Replication Zones](configure-replication-zones.html). + +### When running a multi-node cluster + +When running a multi-node CockroachDB cluster, if you see an error like the one above about replicas failing, some nodes might not be able to talk to each other. For recommended actions, see [Cluster Setup Troubleshooting](cluster-setup-troubleshooting.html#replication-issues). + +## split failed while applying backpressure; are rows updated in a tight loop? + +In CockroachDB, a table row is stored on disk as a key-value pair. Whenever the row is updated, CockroachDB also stores a distinct version of the key-value pair to enable concurrent request processing while guaranteeing consistency (see [multi-version concurrency control (MVCC)](architecture/storage-layer.html#mvcc)). All versions of a key-value pair belong to a larger ["range"](architecture/overview.html#terms) of the total key space, and the historical versions remain until the garbage collection period defined by the `gc.ttlseconds` variable in the applicable [zone configuration](configure-replication-zones.html#gc-ttlseconds) has passed (25 hours by default). Once a range reaches a size threshold (512 MiB by default), CockroachDB splits the range into two ranges. However, this message indicates that a range cannot be split as intended. + +One possible cause is that the range consists only of MVCC version data due to a row being repeatedly updated, and the range cannot be split because doing so would spread MVCC versions for a single row across multiple ranges. + +To resolve this issue, make sure you are not repeatedly updating a single row. If frequent updates of a row are necessary, consider one of the following: + +- Reduce the `gc.ttlseconds` variable in the applicable [zone configuration](configure-replication-zones.html#gc-ttlseconds) to reduce the garbage collection period and prevent such a large build-up of historical values. +- If a row contains large columns that are not being updated with other columns, put the large columns in separate [column families](column-families.html). + +## context deadline exceeded + +This message occurs when a component of CockroachDB gives up because it was relying on another component that has not behaved as expected, for example, another node dropped a network connection. To investigate further, look in the node's logs for the primary failure that is the root cause. + +## result is ambiguous + +In a distributed system, some errors can have ambiguous results. For +example, if you receive a `connection closed` error while processing a +`COMMIT` statement, you cannot tell whether the transaction +successfully committed or not. These errors are possible in any +database, but CockroachDB is somewhat more likely to produce them than +other databases because ambiguous results can be caused by failures +between the nodes of a cluster. These errors are reported with the +PostgreSQL error code `40003` (`statement_completion_unknown`) and the +message `result is ambiguous`. + +Ambiguous errors can be caused by nodes crashing, network failures, or +timeouts. If you experience a lot of these errors when things are +otherwise stable, look for performance issues. Note that ambiguity is +only possible for the last statement of a transaction (`COMMIT` or +`RELEASE SAVEPOINT`) or for statements outside a transaction. If a connection drops during a transaction that has not yet tried to commit, the transaction will definitely be aborted. + +In general, you should handle ambiguous errors the same way as +`connection closed` errors. If your transaction is +[idempotent](https://en.wikipedia.org/wiki/Idempotence#Computer_science_meaning), +it is safe to retry it on ambiguous errors. `UPSERT` operations are +typically idempotent, and other transactions can be written to be +idempotent by verifying the expected state before performing any +writes. Increment operations such as `UPDATE my_table SET x=x+1 WHERE +id=$1` are typical examples of operations that cannot easily be made +idempotent. If your transaction is not idempotent, then you should +decide whether to retry or not based on whether it would be better for +your application to apply the transaction twice or return an error to +the user. + +## Something else? + +If we do not have a solution here, you can try using our other [support resources](support-resources.html), including: + +- [Other troubleshooting pages](troubleshooting-overview.html) +- [StackOverflow](http://stackoverflow.com/questions/tagged/cockroachdb) +- [CockroachDB Community Forum](https://forum.cockroachlabs.com) +- [Chatting with our developers on Gitter](https://gitter.im/cockroachdb/cockroach) (To open Gitter without leaving these docs, click **Help** in the lower-right corner of any page.) diff --git a/v20.2/common-table-expressions.md b/v20.2/common-table-expressions.md new file mode 100644 index 00000000000..14ebaf4022f --- /dev/null +++ b/v20.2/common-table-expressions.md @@ -0,0 +1,320 @@ +--- +title: Common Table Expressions +summary: Common Table Expressions (CTEs) simplify the definition and use of subqueries +toc: true +toc_not_nested: true +--- + +Common Table Expressions, or CTEs, provide a shorthand name to a +possibly complex [subquery](subqueries.html) before it is used in a +larger query context. This improves readability of the SQL code. + +CTEs can be used in combination with [`SELECT` +clauses](select-clause.html) and [`INSERT`](insert.html), +[`DELETE`](delete.html), [`UPDATE`](update.html) and +[`UPSERT`](upsert.html) statements. + + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/with_clause.html %}
+ +
+ +## Parameters + +Parameter | Description +----------|------------ +`table_alias_name` | The name to use to refer to the common table expression from the accompanying query or statement. +`name` | A name for one of the columns in the newly defined common table expression. +`preparable_stmt` | The statement or subquery to use as common table expression. + +## Overview + +{{site.data.alerts.callout_info}} +The examples on this page use MovR, a fictional vehicle-sharing application, to demonstrate CockroachDB SQL statements. To follow along, run [`cockroach demo`](cockroach-demo.html) from the command line to start a temporary, in-memory cluster with the `movr` dataset preloaded. + +For more information about the MovR example application and dataset, see [MovR: A Global Vehicle-sharing App](movr.html). +{{site.data.alerts.end}} + +A query or statement of the form `WITH x AS y IN z` creates the +temporary table name `x` for the results of the subquery `y`, to be +reused in the context of the query `z`. + +For example: + +{% include copy-clipboard.html %} +~~~ sql +> WITH r AS (SELECT * FROM rides WHERE revenue > 98) + SELECT * FROM users AS u, r WHERE r.rider_id = u.id; +~~~ + +~~~ + id | city | name | address | credit_card | id | city | vehicle_city | rider_id | vehicle_id | start_address | end_address | start_time | end_time | revenue +---------------------------------------+---------------+------------------+--------------------------------+-------------+--------------------------------------+---------------+---------------+--------------------------------------+--------------------------------------+-----------------------------------+---------------------------+---------------------------+---------------------------+---------- + ae147ae1-47ae-4800-8000-000000000022 | amsterdam | Tyler Dalton | 88194 Angela Gardens Suite 94 | 4443538758 | bbe76c8b-4395-4000-8000-00000000016f | amsterdam | amsterdam | ae147ae1-47ae-4800-8000-000000000022 | aaaaaaaa-aaaa-4800-8000-00000000000a | 45295 Brewer View Suite 52 | 62188 Jade Causeway | 2018-12-17 03:04:05+00:00 | 2018-12-17 13:04:05+00:00 | 99.00 + c7ae147a-e147-4000-8000-000000000027 | paris | Tina Miller | 97521 Mark Extensions | 8880478663 | d5810624-dd2f-4800-8000-0000000001a1 | paris | paris | c7ae147a-e147-4000-8000-000000000027 | cccccccc-cccc-4000-8000-00000000000c | 47713 Reynolds Mountains Suite 39 | 1417 Stephanie Villages | 2018-12-17 03:04:05+00:00 | 2018-12-18 22:04:05+00:00 | 99.00 + 75c28f5c-28f5-4400-8000-000000000017 | san francisco | William Wood | 36021 Steven Cove Apt. 89 | 5669281259 | 8ac08312-6e97-4000-8000-00000000010f | san francisco | san francisco | 75c28f5c-28f5-4400-8000-000000000017 | 77777777-7777-4800-8000-000000000007 | 84407 Tony Crest | 55336 Jon Manors | 2018-12-10 03:04:05+00:00 | 2018-12-11 13:04:05+00:00 | 99.00 + 8a3d70a3-d70a-4000-8000-00000000001b | san francisco | Jessica Martinez | 96676 Jennifer Knolls Suite 91 | 1601930189 | 7d70a3d7-0a3d-4000-8000-0000000000f5 | san francisco | san francisco | 8a3d70a3-d70a-4000-8000-00000000001b | 77777777-7777-4800-8000-000000000007 | 78978 Stevens Ramp Suite 8 | 7340 Alison Field Apt. 44 | 2018-12-19 03:04:05+00:00 | 2018-12-21 10:04:05+00:00 | 99.00 + 47ae147a-e147-4000-8000-00000000000e | washington dc | Patricia Herrera | 80588 Perez Camp | 6812041796 | 4083126e-978d-4000-8000-00000000007e | washington dc | washington dc | 47ae147a-e147-4000-8000-00000000000e | 44444444-4444-4400-8000-000000000004 | 33055 Julie Dale Suite 93 | 17280 Jill Drives | 2019-01-01 03:04:05+00:00 | 2019-01-01 14:04:05+00:00 | 99.00 +(5 rows) +~~~ + +In this example, the `WITH` clause defines the temporary name `r` for +the subquery over `rides`, and that name becomes a valid table name +for use in any [table expression](table-expressions.html) of the +subsequent `SELECT` clause. + +This query is equivalent to, but arguably simpler to read than: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users AS u, (SELECT * FROM rides WHERE revenue > 98) AS r + WHERE r.rider_id = u.id; +~~~ + +It is also possible to define multiple common table expressions +simultaneously with a single `WITH` clause, separated by commas. Later +subqueries can refer to earlier subqueries by name. For example, the +following query is equivalent to the two examples above: + +{% include copy-clipboard.html %} +~~~ sql +> WITH r AS (SELECT * FROM rides WHERE revenue > 98), + results AS (SELECT * FROM users AS u, r WHERE r.rider_id = u.id) + SELECT * FROM results; +~~~ + +In this example, the second CTE `results` refers to the first CTE `r` +by name. The final query refers to the CTE `results`. + +## Nested `WITH` clauses + +It is possible to use a `WITH` clause in a subquery, or even a `WITH` clause within another `WITH` clause. For example: + +{% include copy-clipboard.html %} +~~~ sql +> WITH u AS + (SELECT * FROM + (WITH u_tab AS (SELECT * FROM users) SELECT * FROM u_tab)) + SELECT * FROM u; +~~~ + +When analyzing [table expressions](table-expressions.html) that +mention a CTE name, CockroachDB will choose the CTE definition that is +closest to the table expression. For example: + +{% include copy-clipboard.html %} +~~~ sql +> WITH + u AS (SELECT * FROM users), + v AS (WITH u AS (SELECT * from vehicles) SELECT * FROM u) + SELECT * FROM v; +~~~ + +In this example, the inner subquery `SELECT * FROM v` will select from +table `vehicles` (closest `WITH` clause), not from table `users`. + +{{site.data.alerts.callout_info}} + CockroachDB does not support nested `WITH` clauses containing [data-modifying statements](#data-modifying-statements). `WITH` clauses containing data-modifying statements must be at the top level of the query. +{{site.data.alerts.end}} + +## Data-modifying statements + +It is possible to use a [data-modifying statement](sql-statements.html#data-manipulation-statements) (`INSERT`, `DELETE`, +etc.) as a common table expression, as long as the `WITH` clause containing the data-modifying statement is at the top level of the query. + +For example: + +{% include copy-clipboard.html %} +~~~ sql +> WITH final_code AS + (INSERT INTO promo_codes(code, description, rules) + VALUES ('half_off', 'Half-price ride!', '{"type": "percent_discount", "value": "50%"}'), ('free_ride', 'Free ride!', '{"type": "percent_discount", "value": "100%"}') + returning rules) + SELECT rules FROM final_code; +~~~ + +~~~ + rules ++-----------------------------------------------+ + {"type": "percent_discount", "value": "50%"} + {"type": "percent_discount", "value": "100%"} +(2 rows) +~~~ + +If the `WITH` clause containing the data-modifying statement is at a lower level, the statement results in an error: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT (WITH final_code AS + (INSERT INTO promo_codes(code, description, rules) + VALUES ('half_off', 'Half-price ride!', '{"type": "percent_discount", "value": "50%"}'), ('free_ride', 'Free ride!', '{"type": "percent_discount", "value": "100%"}') + returning rules) + SELECT rules FROM final_code); +~~~ + +~~~ +ERROR: WITH clause containing a data-modifying statement must be at the top level +SQLSTATE: 0A000 +~~~ + +{{site.data.alerts.callout_info}} +If a common table expression contains +a data-modifying statement (INSERT, DELETE, +etc.), the modifications are performed fully even if only part +of the results are used, e.g., with LIMIT. See Data +Writes in Subqueries for details. +{{site.data.alerts.end}} + +## Reusing common table expressions + +You can reference a CTE multiple times in a single query, using a `WITH` operator. + +For example: + +{% include copy-clipboard.html %} +~~~ sql +> WITH + users_ny AS (SELECT name, id FROM users WHERE city='new york'), + vehicles_ny AS (SELECT type, id, owner_id FROM vehicles WHERE city='new york') + SELECT * FROM users_ny JOIN vehicles_ny ON users_ny.id = vehicles_ny.owner_id; +~~~ + +~~~ + name | id | type | id | owner_id ++------------------+--------------------------------------+------------+--------------------------------------+--------------------------------------+ + James Hamilton | 051eb851-eb85-4ec0-8000-000000000001 | skateboard | 00000000-0000-4000-8000-000000000000 | 051eb851-eb85-4ec0-8000-000000000001 + Catherine Nelson | 147ae147-ae14-4b00-8000-000000000004 | scooter | 11111111-1111-4100-8000-000000000001 | 147ae147-ae14-4b00-8000-000000000004 +(2 rows) +~~~ + +In this single query, you define two CTE's and then reuse them in a table join. + +## Recursive common table expressions + + CockroachDB supports [recursive common table expressions](https://en.wikipedia.org/wiki/Hierarchical_and_recursive_queries_in_SQL#Common_table_expression). Recursive common table expressions are common table expressions that contain subqueries that refer to their own output. + +Recursive CTE definitions take the following form: + +~~~ sql +WITH RECURSIVE () AS ( + + UNION ALL + +) + +~~~ + +To write a recursive CTE: + +1. Add the `RECURSIVE` keyword directly after the `WITH` operator in the CTE definition, and before the CTE name. +1. Define an initial, non-recursive subquery. This subquery defines the initial values of the CTE. +1. Add the `UNION ALL` keyword after the initial subquery. +1. Define a recursive subquery that references its own output. This subquery can also reference the CTE name, unlike the initial subquery. +1. Write a parent query that evaluates the results of the CTE. + +CockroachDB evaluates recursive CTEs as follows: + +1. The initial query is evaluated. Its results are stored to rows in the CTE and copied to a temporary, working table. This working table is updated across iterations of the recursive subquery. +1. The recursive subquery is evaluated iteratively on the contents of the working table. The results of each iteration replace the contents of the working table. The results are also stored to rows of the CTE. The recursive subquery iterates until no results are returned. + +{{site.data.alerts.callout_info}} +Recursive subqueries must eventually return no results, or the query will run indefinitely. +{{site.data.alerts.end}} + +For example, the following recursive CTE calculates the factorial of the numbers 0 through 9: + +{% include copy-clipboard.html %} +~~~ sql +WITH RECURSIVE cte (n, factorial) AS ( + VALUES (0, 1) -- initial subquery + UNION ALL + SELECT n+1, (n+1)*factorial FROM cte WHERE n < 9 -- recursive subquery +) +SELECT * FROM cte; +~~~ + +~~~ + n | factorial ++---+-----------+ + 0 | 1 + 1 | 1 + 2 | 2 + 3 | 6 + 4 | 24 + 5 | 120 + 6 | 720 + 7 | 5040 + 8 | 40320 + 9 | 362880 +(10 rows) +~~~ + +The initial subquery (`VALUES (0, 1)`) initializes the working table with the values `0` for the `n` column and `1` for the `factorial` column. The recursive subquery (`SELECT n+1, (n+1)*factorial FROM cte WHERE n < 9`) evaluates over the initial values of the working table and replaces its contents with the results. It then iterates over the contents of the working table, replacing its contents at each iteration, until `n` reaches `9`, when the [`WHERE` clause](select-clause.html#filter-rows) evaluates as false. + +If no `WHERE` clause were defined in the example, the recursive subquery would always return results and loop indefinitely, resulting in an error: + +{% include copy-clipboard.html %} +~~~ sql +WITH RECURSIVE cte (n, factorial) AS ( + VALUES (0, 1) -- initial subquery + UNION ALL + SELECT n+1, (n+1)*factorial FROM cte -- recursive subquery with no WHERE clause +) +SELECT * FROM cte; +~~~ + +~~~ +ERROR: integer out of range +SQLSTATE: 22003 +~~~ + +If you are unsure if your recursive subquery will loop indefinitely, you can limit the results of the CTE with the [`LIMIT`](limit-offset.html) keyword. + +For example, if we remove the `WHERE` clause from the factorial example, we can use `LIMIT` to limit the results and avoid the `integer out of range` error: + +{% include copy-clipboard.html %} +~~~ sql +WITH RECURSIVE cte (n, factorial) AS ( + VALUES (0, 1) -- initial subquery + UNION ALL + SELECT n+1, (n+1)*factorial FROM cte -- recursive subquery +) +SELECT * FROM cte LIMIT 10; +~~~ + +~~~ + n | factorial ++---+-----------+ + 0 | 1 + 1 | 1 + 2 | 2 + 3 | 6 + 4 | 24 + 5 | 120 + 6 | 720 + 7 | 5040 + 8 | 40320 + 9 | 362880 +(10 rows) +~~~ + +While this practice works for testing and debugging, we do not recommend it in production. + +{{site.data.alerts.callout_info}} +CockroachDB does not currently support the [Postgres recursive CTE variant](https://www.postgresql.org/docs/10/queries-with.html) with the keyword `UNION`. +{{site.data.alerts.end}} + +## Known limitations + +{% include {{ page.version.version }}/known-limitations/correlated-ctes.md %} + +## See also + +- [Subqueries](subqueries.html) +- [Selection Queries](selection-queries.html) +- [Table Expressions](table-expressions.html) +- [`EXPLAIN`](explain.html) diff --git a/v20.2/computed-columns.md b/v20.2/computed-columns.md new file mode 100644 index 00000000000..40c6d40aaf4 --- /dev/null +++ b/v20.2/computed-columns.md @@ -0,0 +1,78 @@ +--- +title: Computed Columns +summary: A computed column stores data generated by an expression included in the column definition. +toc: true +--- + +A computed column stores data generated from other columns by a [scalar expression](scalar-expressions.html) included in the column definition. + + +## Why use computed columns? + +Computed columns are especially useful when used with [partitioning](partitioning.html), [`JSONB`](jsonb.html) columns, or [secondary indexes](indexes.html). + +- **Partitioning** requires that partitions are defined using columns that are a prefix of the [primary key](primary-key.html). In the case of geo-partitioning, some applications will want to collapse the number of possible values in this column, to make certain classes of queries more performant. For example, if a users table has a country and state column, then you can make a stored computed column locality with a reduced domain for use in partitioning. For more information, see the [partitioning example](#create-a-table-with-geo-partitions-and-a-computed-column) below. + +- **JSONB** columns are used for storing semi-structured `JSONB` data. When the table's primary information is stored in `JSONB`, it's useful to index a particular field of the `JSONB` document. In particular, computed columns allow for the following use case: a two-column table with a `PRIMARY KEY` column and a `payload` column, whose primary key is computed as some field from the `payload` column. This alleviates the need to manually separate your primary keys from your JSON blobs. For more information, see the [`JSONB` example](#create-a-table-with-a-jsonb-column-and-a-computed-column) below. + +- **Secondary indexes** can be created on computed columns, which is especially useful when a table is frequently sorted. See the [secondary indexes example](#create-a-table-with-a-secondary-index-on-a-computed-column) below. + +## Considerations + +Computed columns: + +- Cannot be used to generate other computed columns. +- Cannot be a [foreign key](foreign-key.html) reference. +- Behave like any other column, with the exception that they cannot be written to directly. +- Are mutually exclusive with [`DEFAULT`](default-value.html). + +## Creation + +To define a computed column, use the following syntax: + +~~~ +column_name AS () STORED +~~~ + +Parameter | Description +----------|------------ +`column_name` | The [name/identifier](keywords-and-identifiers.html#identifiers) of the computed column. +`` | The [data type](data-types.html) of the computed column. +`` | The pure [scalar expression](scalar-expressions.html) used to compute column values. Any functions marked as `impure`, such as `now()` or `nextval()` cannot be used. +`STORED` | _(Required)_ The computed column is stored alongside other columns. + +## Examples + +### Create a table with a computed column + +{% include {{ page.version.version }}/computed-columns/simple.md %} + +### Create a table with geo-partitions and a computed column + +{% include {{ page.version.version }}/computed-columns/partitioning.md %} The `locality` values can then be used for geo-partitioning. + +### Create a table with a `JSONB` column and a computed column + +{% include {{ page.version.version }}/computed-columns/jsonb.md %} + +### Create a table with a secondary index on a computed column + +{% include {{ page.version.version }}/computed-columns/secondary-index.md %} + +### Add a computed column to an existing table + +{% include {{ page.version.version }}/computed-columns/add-computed-column.md %} + +For more information, see [`ADD COLUMN`](add-column.html). + +### Convert a computed column into a regular column + +{% include {{ page.version.version }}/computed-columns/convert-computed-column.md %} + +## See also + +- [Scalar Expressions](scalar-expressions.html) +- [Information Schema](information-schema.html) +- [`CREATE TABLE`](create-table.html) +- [`JSONB`](jsonb.html) +- [Define Table Partitions (Enterprise)](partitioning.html) diff --git a/v20.2/configure-replication-zones.md b/v20.2/configure-replication-zones.md new file mode 100644 index 00000000000..8c426445479 --- /dev/null +++ b/v20.2/configure-replication-zones.md @@ -0,0 +1,693 @@ +--- +title: Configure Replication Zones +summary: In CockroachDB, you use replication zones to control the number and location of replicas for specific sets of data. +keywords: ttl, time to live, availability zone +toc: true +--- + +Replication zones give you the power to control what data goes where in your CockroachDB cluster. Specifically, they are used to control the number and location of replicas for data belonging to the following objects: + +- Databases +- Tables +- Rows ([enterprise-only](enterprise-licensing.html)) +- Indexes ([enterprise-only](enterprise-licensing.html)) +- All data in the cluster, including internal system data ([via the default replication zone](#view-the-default-replication-zone)) + +For each of the above objects you can control: + +- How many copies of each range to spread through the cluster. +- Which constraints are applied to which data, e.g., "table X's data can only be stored in the German datacenters". +- The maximum size of ranges (how big ranges get before they are split). +- How long old data is kept before being garbage collected. +- Where you would like the leaseholders for certain ranges to be located, e.g., "for ranges that are already constrained to have at least one replica in `region=us-west`, also try to put their leaseholders in `region=us-west`". + +This page explains how replication zones work and how to use the [`CONFIGURE ZONE`](configure-zone.html) statement to manage them. + +{{site.data.alerts.callout_info}} +Currently, only members of the `admin` role can configure replication zones. By default, the `root` user belongs to the `admin` role. +{{site.data.alerts.end}} + +## Overview + +Every [range](architecture/overview.html#glossary) in the cluster is part of a replication zone. Each range's zone configuration is taken into account as ranges are rebalanced across the cluster to ensure that any constraints are honored. + +When a cluster starts, there are two categories of replication zone: + +1. Pre-configured replication zones that apply to internal system data. +2. A single default replication zone that applies to the rest of the cluster. + +You can adjust these pre-configured zones as well as add zones for individual databases, tables, rows, and secondary indexes as needed. Note that adding zones for rows and secondary indexes is [enterprise-only](enterprise-licensing.html). + +For example, you might rely on the [default zone](#view-the-default-replication-zone) to spread most of a cluster's data across all of your datacenters, but [create a custom replication zone for a specific database](#create-a-replication-zone-for-a-database) to make sure its data is only stored in certain datacenters and/or geographies. + +## Replication zone levels + +There are five replication zone levels for [**table data**](architecture/distribution-layer.html#table-data) in a cluster, listed from least to most granular: + +Level | Description +------|------------ +Cluster | CockroachDB comes with a pre-configured `default` replication zone that applies to all table data in the cluster not constrained by a database, table, or row-specific replication zone. This zone can be adjusted but not removed. See [View the Default Replication Zone](#view-the-default-replication-zone) and [Edit the Default Replication Zone](#edit-the-default-replication-zone) for more details. +Database | You can add replication zones for specific databases. See [Create a Replication Zone for a Database](#create-a-replication-zone-for-a-database) for more details. +Table | You can add replication zones for specific tables. See [Create a Replication Zone for a Table](#create-a-replication-zone-for-a-table). +Index ([Enterprise-only](enterprise-licensing.html)) | The [secondary indexes](indexes.html) on a table will automatically use the replication zone for the table. However, with an enterprise license, you can add distinct replication zones for secondary indexes. See [Create a Replication Zone for a Secondary Index](#create-a-replication-zone-for-a-secondary-index) for more details. +Row ([Enterprise-only](enterprise-licensing.html)) | You can add replication zones for specific rows in a table or secondary index by [defining table partitions](partitioning.html). See [Create a Replication Zone for a Table Partition](#create-a-replication-zone-for-a-partition) for more details. + +### For system data + +In addition, CockroachDB stores internal [**system data**](architecture/distribution-layer.html#monolithic-sorted-map-structure) in what are called system ranges. There are two replication zone levels for this internal system data, listed from least to most granular: + +Level | Description +------|------------ +Cluster | The `default` replication zone mentioned above also applies to all system ranges not constrained by a more specific replication zone. +System Range | CockroachDB comes with pre-configured replication zones for important system ranges, such as the "meta" and "liveness" ranges. If necessary, you can add replication zones for the "timeseries" range and other system ranges as well. Editing replication zones for system ranges may override settings from `default`. See [Create a Replication Zone for a System Range](#create-a-replication-zone-for-a-system-range) for more details.

CockroachDB also comes with pre-configured replication zones for the internal `system` database and the `system.jobs` table, which stores metadata about long-running jobs such as schema changes and backups. + +### Level priorities + +When replicating data, whether table or system, CockroachDB always uses the most granular replication zone available. For example, for a piece of user data: + +1. If there's a replication zone for the row, CockroachDB uses it. +2. If there's no applicable row replication zone and the row is from a secondary index, CockroachDB uses the secondary index replication zone. +3. If the row isn't from a secondary index or there is no applicable secondary index replication zone, CockroachDB uses the table replication zone. +4. If there's no applicable table replication zone, CockroachDB uses the database replication zone. +5. If there's no applicable database replication zone, CockroachDB uses the `default` cluster-wide replication zone. + +## Manage replication zones + +Use the [`CONFIGURE ZONE`](configure-zone.html) statement to [add](#create-a-replication-zone-for-a-system-range), [modify](#edit-the-default-replication-zone), [reset](#reset-a-replication-zone), and [remove](#remove-a-replication-zone) replication zones. + +### Replication zone variables + +Use the [`ALTER ... CONFIGURE ZONE`](configure-zone.html) [statement](sql-statements.html) to set a replication zone: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t CONFIGURE ZONE USING range_min_bytes = 0, range_max_bytes = 90000, gc.ttlseconds = 89999, num_replicas = 5, constraints = '[-region=west]'; +~~~ + +{% include {{page.version.version}}/zone-configs/variables.md %} + +### Replication constraints + +The location of replicas, both when they are first added and when they are rebalanced to maintain cluster equilibrium, is based on the interplay between descriptive attributes assigned to nodes and constraints set in zone configurations. + +{{site.data.alerts.callout_success}}For demonstrations of how to set node attributes and replication constraints in different scenarios, see Scenario-based Examples below.{{site.data.alerts.end}} + +#### Descriptive attributes assigned to nodes + +When starting a node with the [`cockroach start`](cockroach-start.html) command, you can assign the following types of descriptive attributes: + +Attribute Type | Description +---------------|------------ +**Node Locality** | Using the [`--locality`](cockroach-start.html#locality) flag, you can assign arbitrary key-value pairs that describe the location of the node. Locality might include region, country, datacenter, rack, etc. The key-value pairs should be ordered into _locality tiers_ that range from most inclusive to least inclusive (e.g., region before datacenter as in `region=eu,dc=paris`), and the keys and the order of key-value pairs must be the same on all nodes. It's typically better to include more pairs than fewer. For example:

`--locality=region=east,datacenter=us-east-1`
`--locality=region=east,datacenter=us-east-2`
`--locality=region=west,datacenter=us-west-1`

CockroachDB attempts to spread replicas evenly across the cluster based on locality, with the order of locality tiers determining the priority. Locality can also be used to influence the location of data replicas in various ways using replication zones.

When there is high latency between nodes, CockroachDB uses locality to move range leases closer to the current workload, reducing network round trips and improving read performance. See [Follow-the-workload](demo-follow-the-workload.html) for more details. +**Node Capability** | Using the `--attrs` flag, you can specify node capability, which might include specialized hardware or number of cores, for example:

`--attrs=ram:64gb` +**Store Type/Capability** | Using the `attrs` field of the `--store` flag, you can specify disk type or capability, for example:

`--store=path=/mnt/ssd01,attrs=ssd`
`--store=path=/mnt/hda1,attrs=hdd:7200rpm` + +#### Types of constraints + +The node-level and store-level descriptive attributes mentioned above can be used as the following types of constraints in replication zones to influence the location of replicas. However, note the following general guidance: + +- When locality is the only consideration for replication, it's recommended to set locality on nodes without specifying any constraints in zone configurations. In the absence of constraints, CockroachDB attempts to spread replicas evenly across the cluster based on locality. +- Required and prohibited constraints are useful in special situations where, for example, data must or must not be stored in a specific country or on a specific type of machine. +- Avoid conflicting constraints. CockroachDB returns an error if you: + - Redefine a required constraint key within the same `constraints` definition on all replicas. For example, `constraints = '[+region=west, +region=east]'` will result in an error. + - Define a required and prohibited definition for the same key-value pair. For example, `constraints = '[-region=west, +region=west]'` will result in an error. + +Constraint Type | Description | Syntax +----------------|-------------|------- +**Required** | When placing replicas, the cluster will consider only nodes/stores with matching attributes or localities. When there are no matching nodes/stores, new replicas will not be added. | `+ssd` +**Prohibited** | When placing replicas, the cluster will ignore nodes/stores with matching attributes or localities. When there are no alternate nodes/stores, new replicas will not be added. | `-ssd` + +#### Scope of constraints + +Constraints can be specified such that they apply to all replicas in a zone or such that different constraints apply to different replicas, meaning you can effectively pick the exact location of each replica. + +Constraint Scope | Description | Syntax +-----------------|-------------|------- +**All Replicas** | Constraints specified using JSON array syntax apply to all replicas in every range that's part of the replication zone. | `constraints = '[+ssd, -region=west]'` +**Per-Replica** | Multiple lists of constraints can be provided in a JSON object, mapping each list of constraints to an integer number of replicas in each range that the constraints should apply to.

The total number of replicas constrained cannot be greater than the total number of replicas for the zone (`num_replicas`). However, if the total number of replicas constrained is less than the total number of replicas for the zone, the non-constrained replicas will be allowed on any nodes/stores.

Note that per-replica constraints must be "required" (e.g., `'{"+region=west": 1}'`); they cannot be "prohibited" (e.g., `'{"-region=west": 1}'`). Also, when defining per-replica constraints on a database or table, `num_replicas` must be specified as well, but not when defining per-replica constraints on an index or partition.

See the [Per-replica constraints](#per-replica-constraints-to-specific-datacenters) example for more details. | `constraints = '{"+ssd,+region=west": 2, "+region=east": 1}', num_replicas = 3` + +### Node/replica recommendations + +See [Cluster Topography](recommended-production-settings.html#topology) recommendations for production deployments. + +### Troubleshooting zone constraint violations + +To see if any of the data placement constraints defined in your replication zone configurations are being violated, use the `system.replication_constraint_stats` report as described in [Replication Reports](query-replication-reports.html). + +## View replication zones + +Use the [`SHOW ZONE CONFIGURATIONS`](#view-all-replication-zones) statement to view details about existing replication zones. + +You can also use the [`SHOW PARTITIONS`](show-partitions.html) statement to view the zone constraints on existing table partitions, or [`SHOW CREATE TABLE`](show-create.html) to view zone configurations for a table. + +{% include {{page.version.version}}/sql/crdb-internal-partitions.md %} + +## Basic examples + +{% include {{ page.version.version }}/sql/movr-statements-geo-partitioned-replicas.md %} + +These examples focus on the basic approach and syntax for working with zone configuration. For examples demonstrating how to use constraints, see [Scenario-based examples](#scenario-based-examples). + +For more examples, see [`CONFIGURE ZONE`](configure-zone.html) and [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html). + +### View all replication zones + +{% include {{ page.version.version }}/zone-configs/view-all-replication-zones.md %} + +For more information, see [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html). + +### View the default replication zone + +{% include {{ page.version.version }}/zone-configs/view-the-default-replication-zone.md %} + +For more information, see [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html). + +### Edit the default replication zone + +{% include {{ page.version.version }}/zone-configs/edit-the-default-replication-zone.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Create a replication zone for a system range + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-system-range.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Create a replication zone for a database + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-database.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Create a replication zone for a table + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-table.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Create a replication zone for a secondary index + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-secondary-index.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Create a replication zone for a partition + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-table-partition.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Reset a replication zone + +{% include {{ page.version.version }}/zone-configs/reset-a-replication-zone.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Remove a replication zone + +{% include {{ page.version.version }}/zone-configs/remove-a-replication-zone.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +### Constrain leaseholders to specific datacenters + +{% include {{ page.version.version }}/zone-configs/constrain-leaseholders-to-specific-datacenters.md %} + +For more information, see [`CONFIGURE ZONE`](configure-zone.html). + +## Scenario-based examples + +### Even replication across datacenters + +**Scenario:** + +- You have 6 nodes across 3 datacenters, 2 nodes in each datacenter. +- You want data replicated 3 times, with replicas balanced evenly across all three datacenters. + +**Approach:** + +1. Start each node with its datacenter location specified in the [`--locality`](cockroach-start.html#locality) flag: + + Datacenter 1: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,, + ~~~ + + Datacenter 2: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,, + ~~~ + + Datacenter 3: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-3 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-3 \ + --join=,, + ~~~ + +2. Initialize the cluster: + + ~~~ shell + $ cockroach init --insecure --host= + ~~~ + +There's no need to make zone configuration changes; by default, the cluster is configured to replicate data three times, and even without explicit constraints, the cluster will aim to diversify replicas across node localities. + +### Per-replica constraints to specific datacenters + +**Scenario:** + +- You have 5 nodes across 5 datacenters in 3 regions, 1 node in each datacenter. +- You want data replicated 3 times, with a quorum of replicas for a database holding West Coast data centered on the West Coast and a database for nation-wide data replicated across the entire country. + +**Approach:** + +1. Start each node with its region and datacenter location specified in the [`--locality`](cockroach-start.html#locality) flag: + + Start the five nodes: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=region=us-west1,datacenter=us-west1-a \ + --join=,,,, + $ cockroach start --insecure --advertise-addr= --locality=region=us-west1,datacenter=us-west1-b \ + --join=,,,, + $ cockroach start --insecure --advertise-addr= --locality=region=us-central1,datacenter=us-central1-a \ + --join=,,,, + $ cockroach start --insecure --advertise-addr= --locality=region=us-east1,datacenter=us-east1-a \ + --join=,,,, + $ cockroach start --insecure --advertise-addr= --locality=region=us-east1,datacenter=us-east1-b \ + --join=,,,, + ~~~ + + Initialize the cluster: + + ~~~ shell + $ cockroach init --insecure --host= + ~~~ + +2. On any node, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +3. Create the database for the West Coast application: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE west_app_db; + ~~~ + +4. Configure a replication zone for the database: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER DATABASE west_app_db + CONFIGURE ZONE USING constraints = '{"+region=us-west1": 2, "+region=us-central1": 1}', num_replicas = 3; + ~~~ + + ~~~ + CONFIGURE ZONE 1 + ~~~ + +5. View the replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR DATABASE west_app_db; + ~~~ + + ~~~ + target | raw_config_sql + +----------------------+--------------------------------------------------------------------+ + DATABASE west_app_db | ALTER DATABASE west_app_db CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 3, + | constraints = '{+region=us-central1: 1, +region=us-west1: 2}', + | lease_preferences = '[]' + (1 row) + ~~~ + + Two of the database's three replicas will be put in `region=us-west1` and its remaining replica will be put in `region=us-central1`. This gives the application the resilience to survive the total failure of any one datacenter while providing low-latency reads and writes on the West Coast because a quorum of replicas are located there. + +6. No configuration is needed for the nation-wide database. The cluster is configured to replicate data 3 times and spread them as widely as possible by default. Because the first key-value pair specified in each node's locality is considered the most significant part of each node's locality, spreading data as widely as possible means putting one replica in each of the three different regions. + +### Multiple applications writing to different databases + +**Scenario:** + +- You have 2 independent applications connected to the same CockroachDB cluster, each application using a distinct database. +- You have 6 nodes across 2 datacenters, 3 nodes in each datacenter. +- You want the data for application 1 to be replicated 5 times, with replicas evenly balanced across both datacenters. +- You want the data for application 2 to be replicated 3 times, with all replicas in a single datacenter. + +**Approach:** + +1. Start each node with its datacenter location specified in the [`--locality`](cockroach-start.html#locality) flag: + + Datacenter 1: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,,,,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,,,,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,,,,, + ~~~ + + Datacenter 2: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,,,,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,,,,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,,,,, + ~~~ + + Initialize the cluster: + + ~~~ shell + $ cockroach init --insecure --host= + ~~~ + +2. On any node, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +3. Create the database for application 1: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE app1_db; + ~~~ + +4. Configure a replication zone for the database used by application 1: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER DATABASE app1_db CONFIGURE ZONE USING num_replicas = 5; + ~~~ + + ~~~ + CONFIGURE ZONE 1 + ~~~ + +5. View the replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR DATABASE app1_db; + ~~~ + + ~~~ + target | raw_config_sql + +------------------+---------------------------------------------+ + DATABASE app1_db | ALTER DATABASE app1_db CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + (1 row) + ~~~ + + Nothing else is necessary for application 1's data. Since all nodes specify their datacenter locality, the cluster will aim to balance the data in the database used by application 1 between datacenters 1 and 2. + +6. Still in the SQL client, create a database for application 2: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE app2_db; + ~~~ + +7. Configure a replication zone for the database used by application 2: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER DATABASE app2_db CONFIGURE ZONE USING constraints = '[+datacenter=us-2]'; + ~~~ + +8. View the replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR DATABASE app2_db; + ~~~ + + ~~~ + target | raw_config_sql + +------------------+---------------------------------------------+ + DATABASE app2_db | ALTER DATABASE app2_db CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 3, + | constraints = '[+datacenter=us-2]', + | lease_preferences = '[]' + (1 row) + ~~~ + + The required constraint will force application 2's data to be replicated only within the `us-2` datacenter. + +### Stricter replication for a table and its secondary indexes + +**Scenario:** + +- You have 7 nodes, 5 with SSD drives and 2 with HDD drives. +- You want data replicated 3 times by default. +- Speed and availability are important for a specific table and its indexes, which are queried very frequently, however, so you want the data in the table and secondary indexes to be replicated 5 times, preferably on nodes with SSD drives. + +**Approach:** + +1. Start each node with `ssd` or `hdd` specified as store attributes: + + 5 nodes with SSD storage: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --store=path=node1,attrs=ssd \ + --join=,, + $ cockroach start --insecure --advertise-addr= --store=path=node2,attrs=ssd \ + --join=,, + $ cockroach start --insecure --advertise-addr= --store=path=node3,attrs=ssd \ + --join=,, + $ cockroach start --insecure --advertise-addr= --store=path=node4,attrs=ssd \ + --join=,, + $ cockroach start --insecure --advertise-addr= --store=path=node5,attrs=ssd \ + --join=,, + ~~~ + + 2 nodes with HDD storage: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --store=path=node6,attrs=hdd \ + --join=,, + $ cockroach start --insecure --advertise-addr= --store=path=node7,attrs=hdd \ + --join=,, + ~~~ + + Initialize the cluster: + + ~~~ shell + $ cockroach init --insecure --host= + ~~~ + +2. On any node, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +3. Create a database and table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE db; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE db.important_table; + ~~~ + +4. Configure a replication zone for the table that must be replicated more strictly: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE db.important_table CONFIGURE ZONE USING num_replicas = 5, constraints = '[+ssd]' + ~~~ + +5. View the replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR TABLE db.important_table; + ~~~ + + ~~~ + target | config_sql + +-------------------------------+---------------------------------------------+ + TABLE db.important_table | ALTER DATABASE app2_db CONFIGURE ZONE USING + | range_min_bytes = 1048576, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[+ssd]', + | lease_preferences = '[]' + (1 row) + ~~~ + + The secondary indexes on the table will use the table's replication zone, so all data for the table will be replicated 5 times, and the required constraint will place the data on nodes with `ssd` drives. + +### Tweaking the replication of system ranges + +**Scenario:** + +- You have nodes spread across 7 datacenters. +- You want data replicated 5 times by default. +- For better performance, you want a copy of the meta ranges in all of the datacenters. +- To save disk space, you only want the internal timeseries data replicated 3 times by default. + +**Approach:** + +1. Start each node with a different [locality](cockroach-start.html#locality) attribute: + + ~~~ shell + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-1 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-2 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-3 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-4 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-5 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-6 \ + --join=,, + $ cockroach start --insecure --advertise-addr= --locality=datacenter=us-7 \ + --join=,, + ~~~ + + Initialize the cluster: + + ~~~ shell + $ cockroach init --insecure --host= + ~~~ + +2. On any node, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure + ~~~ + +3. Configure the default replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE default CONFIGURE ZONE USING num_replicas = 5; + ~~~ + +4. View the replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR RANGE default; + ~~~ + ~~~ + target | raw_config_sql + +---------------+------------------------------------------+ + RANGE default | ALTER RANGE default CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 5, + | constraints = '[]', + | lease_preferences = '[]' + (1 row) + ~~~ + + All data in the cluster will be replicated 5 times, including both SQL data and the internal system data. + +5. Configure the `meta` replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE meta CONFIGURE ZONE USING num_replicas = 7; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR RANGE meta; + ~~~ + ~~~ + target | raw_config_sql + +------------+---------------------------------------+ + RANGE meta | ALTER RANGE meta CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 3600, + | num_replicas = 7, + | constraints = '[]', + | lease_preferences = '[]' + (1 row) + ~~~ + + The `meta` addressing ranges will be replicated such that one copy is in all 7 datacenters, while all other data will be replicated 5 times. + +6. Configure the `timeseries` replication zone: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER RANGE timeseries CONFIGURE ZONE USING num_replicas = 3; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW ZONE CONFIGURATION FOR RANGE timeseries; + ~~~ + ~~~ + target | raw_config_sql + +------------------+---------------------------------------------+ + RANGE timeseries | ALTER RANGE timeseries CONFIGURE ZONE USING + | range_min_bytes = 16777216, + | range_max_bytes = 67108864, + | gc.ttlseconds = 90000, + | num_replicas = 3, + | constraints = '[]', + | lease_preferences = '[]' + (1 row) + ~~~ + + The timeseries data will only be replicated 3 times without affecting the configuration of all other data. + +## See also + +- [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html) +- [`CONFIGURE ZONE`](configure-zone.html) +- [`SHOW PARTITIONS`](show-partitions.html) +- [SQL Statements](sql-statements.html) +- [Table Partitioning](partitioning.html) +- [Replication Reports](query-replication-reports.html) diff --git a/v20.2/configure-zone.md b/v20.2/configure-zone.md new file mode 100644 index 00000000000..9cdc2fd8f2a --- /dev/null +++ b/v20.2/configure-zone.md @@ -0,0 +1,131 @@ +--- +title: CONFIGURE ZONE +summary: Use the CONFIGURE ZONE statement to add, modify, reset, and remove replication zones. +toc: true +--- + +`CONFIGURE ZONE` is a subcommand of the `ALTER DATABASE`, `ALTER TABLE`, `ALTER INDEX`, `ALTER PARTITION`, and `ALTER RANGE` statements and is used to add, modify, reset, or remove [replication zones](configure-replication-zones.html) for those objects. To view details about existing replication zones, see [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html). + +In CockroachDB, you can use **replication zones** to control the number and location of replicas for specific sets of data, both when replicas are first added and when they are rebalanced to maintain cluster equilibrium. + +{{site.data.alerts.callout_info}} +Adding replication zones for secondary indexes and partitions is an [enterprise-only](enterprise-licensing.html) feature. +{{site.data.alerts.end}} + +## Synopsis + +**alter_zone_database_stmt ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/alter_zone_database.html %} +
+ +**alter_zone_table_stmt ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/alter_zone_table.html %} +
+ +**alter_zone_index_stmt ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/alter_zone_index.html %} +
+ +**alter_zone_partition_stmt ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/alter_zone_partition.html %} +
+ +**alter_zone_range_stmt ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/alter_zone_range.html %} +
+ +## Required privileges + +If the target is a [`system` range](#create-a-replication-zone-for-a-system-range), the [`system` database](show-databases.html#preloaded-databases), or a table in the `system` database, the user must be an [`admin`](authorization.html#create-and-manage-roles). For all other databases and tables, the user must have the [CREATE](grant.html#supported-privileges) privilege on the target database or table. + +## Parameters + + Parameter | Description +-----------+------------- +`range_name` | The name of the system [range](architecture/overview.html#glossary) for which to show [replication zone configurations](configure-replication-zones.html). +`database_name` | The name of the [database](create-database.html) for which to show [replication zone configurations](configure-replication-zones.html). +`table_name` | The name of the [table](create-table.html) for which to show [replication zone configurations](configure-replication-zones.html). +`partition_name` | The name of the [partition](partitioning.html) for which to show [replication zone configurations](configure-replication-zones.html). +`index_name` | The name of the [index](indexes.html) for which to show [replication zone configurations](configure-replication-zones.html). +`variable` | The name of the [variable](#variables) to change. +`value` | The value of the variable to change. +`DISCARD` | Remove a replication zone. + +### Variables + +{% include {{ page.version.version }}/zone-configs/variables.md %} + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +{% include {{ page.version.version }}/sql/movr-statements-geo-partitioned-replicas.md %} + +### Edit a replication zone + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users CONFIGURE ZONE USING range_min_bytes = 0, range_max_bytes = 90000, gc.ttlseconds = 89999, num_replicas = 4; +~~~ + +~~~ +CONFIGURE ZONE 1 +~~~ + +### Edit the default replication zone + +{% include {{ page.version.version }}/zone-configs/edit-the-default-replication-zone.md %} + +### Create a replication zone for a database + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-database.md %} + +### Create a replication zone for a table + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-table.md %} + +### Create a replication zone for a secondary index + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-secondary-index.md %} + +### Create a replication zone for a partition + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-table-partition.md %} + +### Create a replication zone for a system range + +{% include {{ page.version.version }}/zone-configs/create-a-replication-zone-for-a-system-range.md %} + +### Reset a replication zone + +{% include {{ page.version.version }}/zone-configs/reset-a-replication-zone.md %} + +### Remove a replication zone + +{% include {{ page.version.version }}/zone-configs/remove-a-replication-zone.md %} + +## See also + +- [Configure Replication Zones](configure-replication-zones.html) +- [`PARTITION BY`](partition-by.html) +- [`SHOW ZONE CONFIGURATIONS`](show-zone-configurations.html) +- [`ALTER DATABASE`](alter-database.html) +- [`ALTER TABLE`](alter-table.html) +- [`ALTER INDEX`](alter-index.html) +- [`ALTER PARTITION`](alter-partition.html) +- [`ALTER RANGE`](alter-range.html) +- [`SHOW JOBS`](show-jobs.html) +- [Table Partitioning](partitioning.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/connect-to-the-database.md b/v20.2/connect-to-the-database.md new file mode 100644 index 00000000000..65a148981f8 --- /dev/null +++ b/v20.2/connect-to-the-database.md @@ -0,0 +1,135 @@ +--- +title: Connect to the Database +summary: How to connect to a CockroachDB cluster from your application +toc: true +--- + +This page has instructions for connecting to a CockroachDB cluster from your application using various programming languages. Each example shows a [connection string][connection_params] for a [secure local cluster][local_secure] to a `bank` database by a user named `maxroach`. Depending on your cluster's configuration, you may need to edit this connection string. + +For a reference that lists all of the supported cluster connection parameters, see [Connection Parameters][connection_params]. + +## Before you begin + +Make sure you have already: + +- Set up a [local cluster](secure-a-cluster.html). +- [Installed a Postgres client](install-client-drivers.html). + +## Connect + +
+ + + + +
+ +
+ +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --certs-dir=certs --host=localhost:26257 +~~~ + +For more information about how to use the built-in SQL client, see the [`cockroach sql`](cockroach-sql.html) reference docs. + +
+ +
+ +{% include copy-clipboard.html %} +~~~ go +import ( + "database/sql" + "fmt" + "log" + _ "github.com/lib/pq" +) + +db, err := sql.Open("postgres", + "postgresql://maxroach@localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.maxroach.key&sslcert=certs/client.maxroach.crt") +if err != nil { + log.Fatal("error connecting to the database: ", err) +} +defer db.Close() +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-go.md %} + +
+ +
+ +{% include copy-clipboard.html %} +~~~ java +import java.sql.*; +import javax.sql.DataSource; + +PGSimpleDataSource ds = new PGSimpleDataSource(); +ds.setServerName("localhost"); +ds.setPortNumber(26257); +ds.setDatabaseName("bank"); +ds.setUser("maxroach"); +ds.setPassword(null); +ds.setSsl(true); +ds.setSslMode("require"); +ds.setSslCert("certs/client.maxroach.crt"); +ds.setSslKey("certs/client.maxroach.key.pk8"); +ds.setReWriteBatchedInserts(true); // add `rewriteBatchedInserts=true` to pg connection string +ds.setApplicationName("BasicExample"); +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-java.md %} + +
+ +
+ +{% include copy-clipboard.html %} +~~~ python +import psycopg2 + +conn = psycopg2.connect( + database='bank', + user='maxroach', + sslmode='require', + sslrootcert='certs/ca.crt', + sslkey='certs/client.maxroach.key', + sslcert='certs/client.maxroach.crt', + port=26257, + host='localhost' +) +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-python.md %} + +
+ +## See also + +Reference information related to this task: + +- [Connection parameters][connection_params] +- [Manual deployments][manual] +- [Orchestrated deployments][orchestrated] +- [Start a local cluster (secure)][local_secure] + + + +Other common tasks: + +- [Insert Data](insert-data.html) +- [Query Data](query-data.html) +- [Update Data](update-data.html) +- [Delete Data](delete-data.html) +- [Run Multi-Statement Transactions](run-multi-statement-transactions.html) +- [Error Handling and Troubleshooting](error-handling-and-troubleshooting.html) +- [Make Queries Fast](make-queries-fast.html) +- [Hello World Example apps](hello-world-example-apps.html) + + + +[manual]: manual-deployment.html +[orchestrated]: orchestration.html +[local_secure]: secure-a-cluster.html +[connection_params]: connection-parameters.html diff --git a/v20.2/connection-parameters.md b/v20.2/connection-parameters.md new file mode 100644 index 00000000000..cf959b4ad1e --- /dev/null +++ b/v20.2/connection-parameters.md @@ -0,0 +1,243 @@ +--- +title: Client Connection Parameters +summary: This page describes the parameters used to establish a client connection. +toc: true +--- + +Client applications, including [`cockroach` client +commands](cockroach-commands.html), work by establishing a network +connection to a CockroachDB cluster. The client connection parameters +determine which CockroachDB cluster they connect to, and how to +establish this network connection. + +## Supported connection parameters + +Most client apps, including `cockroach` client commands, determine +which CockroachDB server to connect to using a [PostgreSQL connection +URL](#connect-using-a-url). When using a URL, a client can also +specify additional SQL-level parameters. This mode provides the most +configuration flexibility. + +In addition, all `cockroach` client commands also accept [discrete +connection parameters](#connect-using-discrete-parameters) that can +specify the connection parameters separately from a URL. + +## When to use a URL and when to use discrete parameters + +Specifying client parameters using a URL may be more convenient during +experimentation, as it facilitates copy-pasting the connection +parameters (the URL) between different tools: the output of `cockroach +start`, other `cockroach` commands, GUI database visualizer, +programming tools, etc. + + +Discrete parameters may be more convenient in automation, where the +components of the configuration are filled in separately from +different variables in a script or a service manager. + +## Connect using a URL + +A connection URL has the following format: + +~~~ +postgres://:@:/? +~~~ + + `cockroach` client commands also support [UNIX domain socket URIs](https://en.wikipedia.org/wiki/Unix_domain_socket) of the following form: + +~~~ +postgres://:@?host=&port=& +~~~ + + Component | Description | Required +----------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------- + `` | The [SQL user](create-user.html) that will own the client session. | ✗ + `` | The user's password. It is not recommended to pass the password in the URL directly.

[Find more detail about how CockroachDB handles passwords](authentication.html#client-authentication). | ✗ + `` | The host name or address of a CockroachDB node or load balancer. | Required by most client drivers. + `` | The port number of the SQL interface of the CockroachDB node or load balancer. The default port number for CockroachDB is 26257. Use this value when in doubt. | Required by most client drivers. + `` | A database name to use as [current database](sql-name-resolution.html#current-database). Defaults to `defaultdb`. | ✗ + `` | The directory path to the client listening for a socket connection. | Required when specifying a Unix domain socket URI. + `` | [Additional connection parameters](#additional-connection-parameters), including SSL/TLS certificate settings. | ✗ + + +{{site.data.alerts.callout_info}} +For cockroach commands that accept a URL, you can specify the URL with the command-line flag `--url`. +If `--url` is not specified but +the environment variable `COCKROACH_URL` is defined, the environment +variable is used. Otherwise, the `cockroach` command will use +[discrete connection parameters](#connect-using-discrete-parameters) +as described below. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +The `` part is not used for [`cockroach` +commands](cockroach-commands.html) other than [`cockroach +sql`](cockroach-sql.html). A warning +is currently printed if it is mistakenly specified, and +future versions of CockroachDB may return an error in that case. +{{site.data.alerts.end}} + +### Additional connection parameters + +The following additional parameters can be passed after the `?` character in the URL: + +Parameter | Description | Default value +----------|-------------|--------------- +`application_name` | An initial value for the [`application_name` session variable](set-vars.html).

Note: For [Java JBDC](build-a-java-app-with-cockroachdb.html), use `ApplicationName`. | Empty string. +`sslmode` | Which type of secure connection to use: `disable`, `allow`, `prefer`, `require`, `verify-ca` or `verify-full`. See [Secure Connections With URLs](#secure-connections-with-urls) for details. | `disable` +`sslrootcert` | Path to the [CA certificate](cockroach-cert.html), when `sslmode` is not `disable`. | Empty string. +`sslcert` | Path to the [client certificate](cockroach-cert.html), when `sslmode` is not `disable`. | Empty string. +`sslkey` | Path to the [client private key](cockroach-cert.html), when `sslmode` is not `disable`. | Empty string. + +### Secure connections with URLs + +The following values are supported for `sslmode`, although only the first and the last are recommended for use. + +Parameter | Description | Recommended for use +----------|-------------|-------------------- +`sslmode=disable` | Do not use an encrypted, secure connection at all. | Use during development. +`sslmode=allow` | Enable a secure connection only if the server requires it.

**Not supported in all clients.** | +`sslmode=prefer` | Try to establish a secure connection, but accept an insecure connection if the server does not support secure connections.

**Not supported in all clients.** | +`sslmode=require` | Force a secure connection. An error occurs if the secure connection cannot be established. | +`sslmode=verify-ca` | Force a secure connection and verify that the server certificate is signed by a known CA. | +`sslmode=verify-full` | Force a secure connection, verify that the server certificate is signed by a known CA, and verify that the server address matches that specified in the certificate. | Use for [secure deployments](secure-a-cluster.html). + +{{site.data.alerts.callout_danger}} +Some client drivers and the `cockroach` commands do not support +`sslmode=allow` and `sslmode=prefer`. Check the documentation of your +SQL driver to determine whether these options are supported. +{{site.data.alerts.end}} + +### Example URL for an insecure connection + +The following URL is suitable to connect to a CockroachDB node using an insecure connection: + +~~~ +postgres://root@servername:26257/mydb?sslmode=disable +~~~ + +This specifies a connection for the `root` user to server `servername` +on port 26257 (the default CockroachDB SQL port), with `mydb` set as +current database. `sslmode=disable` makes the connection insecure. + +### Example URL for a secure connection + +The following URL is suitable to connect to a CockroachDB node using a secure connection: + +~~~ +postgres://root@servername:26257/mydb?sslmode=verify-full&sslrootcert=path/to/ca.crt&sslcert=path/to/client.username.crt&sslkey=path/to/client.username.key +~~~ + +This uses the following components: + +- User `root` +- Host name `servername`, port number 26257 (the default CockroachDB SQL port) +- Current database `mydb` +- SSL/TLS mode `verify-full`: + - Root CA certificate `path/to/ca.crt` + - Client certificate `path/to/client.username.crt` + - Client key `path/to/client.username.key` + +For details about how to create and manage SSL/TLS certificates, see +[Create Security Certificates](cockroach-cert.html) and +[Rotate Certificates](rotate-certificates.html). + +### Example URI for a Unix domain socket + + The following URI is suitable to connect to a CockroachDB cluster listening for Unix domain socket connections at `/path/to/client`: + +~~~ +postgres://root@?host=/path/to/client&port=26257 +~~~ + +This specifies a connection for the `root` user to an insecure cluster listening for a socket connection (e.g., a cluster started with the [`--socket-dir` flag](cockroach-start.html#networking)) at `/path/to/client`, and on port 26257. + +## Connect using discrete parameters + +Most [`cockroach` commands](cockroach-commands.html) accept connection +parameters as separate, discrete command-line flags, in addition (or +in replacement) to `--url` which [specifies all parameters as a +URL](#connect-using-a-url). + +For each command-line flag that directs a connection parameter, +CockroachDB also recognizes an environment variable. The environment +variable is used when the command-line flag is not specified. + +{% include {{ page.version.version }}/sql/connection-parameters.md %} + +### Example command-line flags for an insecure connection + +The following command-line flags establish an insecure connection: + +~~~ +--user=root \ +--host= +--insecure +~~~ + +This specifies a connection for the `root` user to server `servername` +on port 26257 (the default CockroachDB SQL port). `--insecure` makes +the connection insecure. + +### Example command-line flags for a secure connection + +The following command-line flags establish a secure connection: + +~~~ +--user=root \ +--host= +--certs-dir=path/to/certs +~~~ + +This uses the following components: + +- User `root` +- Host name `servername`, port number 26257 (the default CockroachDB SQL port) +- SSL/TLS enabled, with settings: + - Root CA certificate `path/to/certs/ca.crt` + - Client certificate `path/to/client..crt` (`path/to/certs/client.root.crt` with `--user root`) + - Client key `path/to/client..key` (`path/to/certs/client.root.key` with `--user root`) + +{{site.data.alerts.callout_info}} +When using discrete connection parameters, the file names of the CA +and client certificates and client key are derived automatically from +the value of `--certs-dir`. +{{site.data.alerts.end}} + +## Using both URL and client parameters + +Most `cockroach` commands accept both a URL and client parameters. +The information contained therein is combined in the order it appears +in the command line. + +This combination is useful so that discrete command-line flags can +override settings not otherwise set in the URL. + +### Example override of the current database + +The `cockroach start` command prints out the following connection URL, which connects to the `defaultdb` database: + +~~~ +postgres://root@servername:26257/?sslmode=disable +~~~ + +To specify `mydb` as the current database using [`cockroach sql`](cockroach-sql.html), run the following command: + +~~~ +cockroach sql \ +--url "postgres://root@servername:26257/?sslmode=disable" \ +--database mydb +~~~ + +This is equivalent to: + +~~~ +cockroach sql --url "postgres://root@servername:26257/mydb?sslmode=disable" +~~~ + +## See also + +- [`cockroach` commands](cockroach-commands.html) +- [Create Security Certificates](cockroach-cert.html) +- [Secure a Cluster](secure-a-cluster.html) +- [Create and Manage Users](authorization.html#create-and-manage-users) diff --git a/v20.2/constraints.md b/v20.2/constraints.md new file mode 100644 index 00000000000..e0a8f79a0f0 --- /dev/null +++ b/v20.2/constraints.md @@ -0,0 +1,125 @@ +--- +title: Constraints +summary: Constraints offer additional data integrity by enforcing conditions on the data within a column. +toc: true +--- + +Constraints offer additional data integrity by enforcing conditions on the data within a column. Whenever values are manipulated (inserted, deleted, or updated), constraints are checked and modifications that violate constraints are rejected. + +For example, the `UNIQUE` constraint requires that all values in a column be unique from one another (except *NULL* values). If you attempt to write a duplicate value, the constraint rejects the entire statement. + + +## Supported constraints + + Constraint | Description +------------|------------- + [`CHECK`](check.html) | Values must return `TRUE` or `NULL` for a Boolean expression. + [`DEFAULT` value](default-value.html) | If a value is not defined for the constrained column in an `INSERT` statement, the `DEFAULT` value is written to the column. + [`FOREIGN KEY`](foreign-key.html) | Values must exactly match existing values from the column it references. + [`NOT NULL`](not-null.html) | Values may not be *NULL*. + [`PRIMARY KEY`](primary-key.html) | Values must uniquely identify each row *(one per table)*. This behaves as if the `NOT NULL` and `UNIQUE` constraints are applied, as well as automatically creates an [index](indexes.html) for the table using the constrained columns. + [`UNIQUE`](unique.html) | Each non-*NULL* value must be unique. This also automatically creates an [index](indexes.html) for the table using the constrained columns. + +## Using constraints + +### Add constraints + +How you add constraints depends on the number of columns you want to constrain, as well as whether or not the table is new. + +- **One column of a new table** has its constraints defined after the column's data type. For example, this statement applies the `PRIMARY KEY` constraint to `foo.a`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE foo (a INT PRIMARY KEY); + ~~~ +- **Multiple columns of a new table** have their constraints defined after the table's columns. For example, this statement applies the `PRIMARY KEY` constraint to `foo`'s columns `a` and `b`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bar (a INT, b INT, PRIMARY KEY (a,b)); + ~~~ + + {{site.data.alerts.callout_info}} + The `DEFAULT` and `NOT NULL` constraints cannot be applied to multiple columns. + {{site.data.alerts.end}} + +- **Existing tables** can have the following constraints added: + - `CHECK`, `FOREIGN KEY`, and `UNIQUE` constraints can be added through [`ALTER TABLE...ADD CONSTRAINT`](add-constraint.html). For example, this statement adds the `UNIQUE` constraint to `baz.id`: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE baz ADD CONSTRAINT id_unique UNIQUE (id); + ~~~ + + - `DEFAULT` values and `NOT NULL` constraints can be added through [`ALTER TABLE...ALTER COLUMN`](alter-column.html#set-or-change-a-default-value). For example, this statement adds the Default Value constraint to `baz.bool`: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE baz ALTER COLUMN bool SET DEFAULT true; + ~~~ + + - [`PRIMARY KEY`](primary-key.html) constraints can be added with [`ADD CONSTRAINT`](add-constraint.html)/[`ADD PRIMARY KEY`](alter-table.html) in the following circumstances: + + - A [`DROP CONSTRAINT`](drop-constraint.html) statement precedes the `ADD CONSTRAINT`/`ADD PRIMARY KEY` statement in the same transaction. For examples, see the [`ADD CONSTRAINT`](add-constraint.html#examples) and [`DROP CONSTRAINT`](drop-constraint.html#examples) pages. + - The current [primary key is on `rowid`](indexes.html#creation), the default primary key created if none is explicitly defined at table creation. + - The `ADD CONSTRAINT`/`ADD PRIMARY KEY` is in the same transaction as a `CREATE TABLE` statement with no primary key defined. + +#### Order of constraints + +The order in which you list constraints is not important because constraints are applied to every modification of their respective tables or columns. + +#### Name constraints on new tables + +You can name constraints applied to new tables using the `CONSTRAINT` clause before defining the constraint: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE foo (a INT CONSTRAINT another_name PRIMARY KEY); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE bar (a INT, b INT, CONSTRAINT yet_another_name PRIMARY KEY (a,b)); +~~~ + +### View constraints + +To view a table's constraints, use [`SHOW CONSTRAINTS`](show-constraints.html) or [`SHOW CREATE`](show-create.html). + +### Remove constraints + +The procedure for removing a constraint depends on its type: + +Constraint Type | Procedure +-----------------|----------- +[`CHECK`](check.html) | Use [`DROP CONSTRAINT`](drop-constraint.html). +[`DEFAULT` value](default-value.html) | Use [`ALTER COLUMN`](alter-column.html#remove-default-constraint). +[`FOREIGN KEY`](foreign-key.html) | Use [`DROP CONSTRAINT`](drop-constraint.html). +[`NOT NULL`](not-null.html) | Use [`ALTER COLUMN`](alter-column.html#remove-not-null-constraint). +[`PRIMARY KEY`](primary-key.html) | Primary key constraints can be dropped with [`DROP CONSTRAINT`](drop-constraint.html) if an [`ADD CONSTRAINT`](add-constraint.html) statement follows the `DROP CONSTRAINT` statement in the same transaction. +[`UNIQUE`](unique.html) | The `UNIQUE` constraint cannot be dropped directly. To remove the constraint, [drop the index](drop-index.html) that was created by the constraint, e.g., `DROP INDEX my_unique_constraint`. + +### Change constraints + +The procedure for changing a constraint depends on its type: + +Constraint Type | Procedure +-----------------|----------- +[`CHECK`](check.html) | [Issue a transaction](transactions.html#syntax) that adds a new `CHECK` constraint ([`ADD CONSTRAINT`](add-constraint.html)), and then remove the existing one ([`DROP CONSTRAINT`](drop-constraint.html)). +[`DEFAULT` value](default-value.html) | The `DEFAULT` value can be changed through [`ALTER COLUMN`](alter-column.html). +[`FOREIGN KEY`](foreign-key.html) | [Issue a transaction](transactions.html#syntax) that adds a new `FOREIGN KEY` constraint ([`ADD CONSTRAINT`](add-constraint.html)), and then remove the existing one ([`DROP CONSTRAINT`](drop-constraint.html)). +[`NOT NULL`](not-null.html) | The `NOT NULL` constraint cannot be changed, only added and removed with [`ALTER COLUMN`](alter-column.html). +[`PRIMARY KEY`](primary-key.html) | To change a primary key, use an [`ALTER TABLE ... ALTER PRIMARY KEY`](alter-primary-key.html) statement.

When you change a primary key with [`ALTER PRIMARY KEY`](alter-primary-key.html), the old primary key index becomes a secondary index. If you do not want the old primary key to become a secondary index, use [`DROP CONSTRAINT`](drop-constraint.html)/[`ADD CONSTRAINT`](add-constraint.html) to change the primary key. +[`UNIQUE`](unique.html) | [Issue a transaction](transactions.html#syntax) that adds a new `UNIQUE` constraint ([`ADD CONSTRAINT`](add-constraint.html)), and then remove the existing one ([`DROP CONSTRAINT`](drop-constraint.html)). + + +## See also + +- [`CREATE TABLE`](create-table.html) +- [`ADD CONSTRAINT`](add-constraint.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) +- [`SHOW CREATE`](show-create.html) +- [`ALTER PRIMARY KEY`](alter-primary-key.html) +- [`ALTER TABLE`](alter-table.html) +- [`ALTER COLUMN`](alter-column.html) diff --git a/v20.2/copy.pl b/v20.2/copy.pl new file mode 100644 index 00000000000..deb722199c9 --- /dev/null +++ b/v20.2/copy.pl @@ -0,0 +1,122 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use feature qw/ say /; + +my @files = qw/ + create-table-as.md + delete.md + export.md + import.md + import-into.md + insert.md + select-clause.md + selection-queries.md + truncate.md + update.md + upsert.md + selection-queries.md + add-column.md + add-constraint.md + alter-column.md + alter-database.md + alter-index.md + alter-partition.md + alter-range.md + alter-sequence.md + alter-table.md + alter-type.md + alter-user.md + alter-view.md + comment-on.md + configure-zone.md + create-database.md + create-index.md + create-sequence.md + create-table.md + create-table-as.md + create-view.md + drop-column.md + drop-constraint.md + drop-database.md + drop-index.md + drop-sequence.md + drop-table.md + drop-view.md + experimental-audit.md + partition-by.md + rename-column.md + rename-constraint.md + rename-database.md + rename-index.md + rename-sequence.md + rename-table.md + show-columns.md + show-constraints.md + show-create.md + show-databases.md + show-partitions.md + show-index.md + show-locality.md + show-schemas.md + show-sequences.md + show-tables.md + show-ranges.md + show-zone-configurations.md + split-at.md + unsplit-at.md + validate-constraint.md + begin-transaction.md + commit-transaction.md + release-savepoint.md + rollback-transaction.md + savepoint.md + set-transaction.md + show-vars.md + create-role.md + create-user.md + drop-role.md + drop-user.md + grant.md + grant-roles.md + revoke.md + revoke-roles.md + show-grants.md + show-roles.md + show-users.md + reset-vars.md + set-vars.md + set-transaction.md + show-trace.md + show-vars.md + reset-cluster-setting.md + set-cluster-setting.md + show-cluster-setting.md + show-sessions.md + cancel-session.md + cancel-query.md + show-queries.md + create-statistics.md + explain.md + explain-analyze.md + show-statistics.md + cancel-job.md + pause-job.md + resume-job.md + show-jobs.md + enterprise-licensing.md + backup.md + backup.md + restore.md + show-backup.md + change-data-capture.md + create-changefeed.md + changefeed-for.md + /; + +for my $file (@files) { + my $target_dir = qq[/Users/rloveland/work/code/sqlchecker/data/]; + my $cmd = qq[cp $file $target_dir]; + system $cmd; +} diff --git a/v20.2/cost-based-optimizer.md b/v20.2/cost-based-optimizer.md new file mode 100644 index 00000000000..1423d2eb868 --- /dev/null +++ b/v20.2/cost-based-optimizer.md @@ -0,0 +1,623 @@ +--- +title: Cost-Based Optimizer +summary: Learn about the cost-based optimizer +toc: true +redirect_from: sql-optimizer.html +--- + +The cost-based optimizer seeks the lowest cost for a query, usually related to time. + +## How is cost calculated? + +A given SQL query can have thousands of equivalent query plans with vastly different execution times. The cost-based optimizer enumerates these plans and chooses the lowest cost plan. + +Cost is roughly calculated by: + +- Estimating how much time each node in the query plan will use to process all results +- Modeling how data flows through the query plan + +The most important factor in determining the quality of a plan is cardinality (i.e., the number of rows); the fewer rows each SQL operator needs to process, the faster the query will run. + +## Table statistics + +The cost-based optimizer can often find more performant query plans if it has access to statistical data on the contents of your tables. This data needs to be generated from scratch for new tables, and regenerated periodically for existing tables. + +By default, CockroachDB generates table statistics automatically when tables are [created](create-table.html), and as they are [updated](update.html). It does this [using a background job](create-statistics.html#view-statistics-jobs) that automatically determines which columns to get statistics on — specifically, it chooses: + +- Columns that are part of the primary key or an index (in other words, all indexed columns). +- Up to 100 non-indexed columns. + +{{site.data.alerts.callout_info}} +[Schema changes](online-schema-changes.html) trigger automatic statistics collection for the affected table(s). +{{site.data.alerts.end}} + +### Controlling automatic statistics + +For best query performance, most users should leave automatic statistics enabled with the default settings. The information provided in this section is useful for troubleshooting or performance tuning by advanced users. + +#### Controlling statistics refresh rate + +Statistics are refreshed in the following cases: + +1. When there are no statistics. +2. When it's been a long time since the last refresh, where "long time" is defined according to a moving average of the time across the last several refreshes. +3. After each mutation operation ([`INSERT`](insert.html), [`UPDATE`](update.html), or [`DELETE`](delete.html)), the probability of a refresh is calculated using a formula that takes the [cluster settings](cluster-settings.html) shown below as inputs. These settings define the target number of rows in a table that should be stale before statistics on that table are refreshed. Increasing either setting will reduce the frequency of refreshes. In particular, `min_stale_rows` impacts the frequency of refreshes for small tables, while `fraction_stale_rows` has more of an impact on larger tables. + +| Setting | Default Value | Details | +|------------------------------------------------------+---------------+--------------------------------------------------------------------------------------| +| `sql.stats.automatic_collection.fraction_stale_rows` | 0.2 | Target fraction of stale rows per table that will trigger a statistics refresh | +| `sql.stats.automatic_collection.min_stale_rows` | 500 | Target minimum number of stale rows per table that will trigger a statistics refresh | + +{{site.data.alerts.callout_info}} +Because the formula for statistics refreshes is probabilistic, you should not expect to see statistics update immediately after changing these settings, or immediately after exactly 500 rows have been updated. +{{site.data.alerts.end}} + +#### Turning off statistics + +If you need to turn off automatic statistics collection, follow the steps below: + +1. Run the following statement to disable the automatic statistics [cluster setting](cluster-settings.html): + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING sql.stats.automatic_collection.enabled = false; + ~~~ + +2. Use the [`SHOW STATISTICS`](show-statistics.html) statement to view automatically generated statistics. + +3. Delete the automatically generated statistics using the following statement: + + {% include copy-clipboard.html %} + ~~~ sql + > DELETE FROM system.table_statistics WHERE true; + ~~~ + +4. Restart the nodes in your cluster to clear the statistics caches. + +For instructions showing how to manually generate statistics, see the examples in the [`CREATE STATISTICS` documentation](create-statistics.html). + +#### Controlling histogram collection + +By default, the optimizer collects histograms for all index columns (specifically the first column in each index) during automatic statistics collection. If a single column statistic is explicitly requested using manual invocation of [`CREATE STATISTICS`](create-statistics.html), a histogram will be collected, regardless of whether or not the column is part of an index. + +If you are an advanced user and need to disable histogram collection for troubleshooting or performance tuning reasons, change the [`sql.stats.histogram_collection.enabled` cluster setting](cluster-settings.html) by running [`SET CLUSTER SETTING`](set-cluster-setting.html) as follows: + +{% include copy-clipboard.html %} +~~~ sql +SET CLUSTER SETTING sql.stats.histogram_collection.enabled = false; +~~~ + +{{site.data.alerts.callout_info}} +When `sql.stats.histogram_collection.enabled` is set to `false`, histograms are never collected, either as part of automatic statistics collection or by manual invocation of [`CREATE STATISTICS`](create-statistics.html). +{{site.data.alerts.end}} + +## Query plan cache + +CockroachDB uses a cache for the query plans generated by the optimizer. This can lead to faster query execution since the database can reuse a query plan that was previously calculated, rather than computing a new plan each time a query is executed. + +The query plan cache is enabled by default. To disable it, execute the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING sql.query_cache.enabled = false; +~~~ + +Finally, note that only the following statements use the plan cache: + +- [`SELECT`](select.html) +- [`INSERT`](insert.html) +- [`UPDATE`](update.html) +- [`UPSERT`](upsert.html) +- [`DELETE`](delete.html) + +## Join reordering + +The cost-based optimizer will explore additional join orderings in an attempt to find the lowest-cost execution plan for a query involving multiple joins, which can lead to significantly better performance in some cases. + +Because this process leads to an exponential increase in the number of possible execution plans for such queries, it's only used to reorder subtrees containing 4 or fewer joins by default. + +To change this setting, which is controlled by the `reorder_joins_limit` [session variable](set-vars.html), run the statement shown below. To disable this feature, set the variable to `0`. + +{% include copy-clipboard.html %} +~~~ sql +> SET reorder_joins_limit = 6; +~~~ + +{{site.data.alerts.callout_danger}} +We strongly recommend not setting this value higher than 8 to avoid performance degradation. If set too high, the cost of generating and costing execution plans can end up dominating the total execution time of the query. +{{site.data.alerts.end}} + +For more information about the difficulty of selecting an optimal join ordering, see our blog post [An Introduction to Join Ordering](https://www.cockroachlabs.com/blog/join-ordering-pt1/). + +## Join hints + +The optimizer supports hint syntax to force the use of a specific join algorithm. The algorithm is specified between the join type (`INNER`, `LEFT`, etc.) and the `JOIN` keyword, for example: + +- `INNER HASH JOIN` +- `OUTER MERGE JOIN` +- `LEFT LOOKUP JOIN` +- `CROSS MERGE JOIN` + +Note that the hint cannot be specified with a bare hint keyword (e.g., `MERGE`) - in that case, the `INNER` keyword must be added. For example, `a INNER MERGE JOIN b` will work, but `a MERGE JOIN b` will not work. + +{{site.data.alerts.callout_info}} +Join hints cannot be specified with a bare hint keyword (e.g., `MERGE`) due to SQL's implicit `AS` syntax. If you're not careful, you can make `MERGE` be an alias for a table; for example, `a MERGE JOIN b` will be interpreted as having an implicit `AS` and be executed as `a AS MERGE JOIN b`, which is just a long way of saying `a JOIN b`. Because the resulting query might execute without returning any hint-related error (because it is valid SQL), it will seem like the join hint "worked", but actually it didn't affect which join algorithm was used. In this case, the correct syntax is `a INNER MERGE JOIN b`. +{{site.data.alerts.end}} + +### Supported join algorithms + +- `HASH`: Forces a hash join; in other words, it disables merge and lookup joins. A hash join is always possible, even if there are no equality columns - CockroachDB considers the nested loop join with no index a degenerate case of the hash join (i.e., a hash table with one bucket). + +- `MERGE`: Forces a merge join, even if it requires re-sorting both sides of the join. + +- `LOOKUP`: Forces a lookup join into the right side; the right side must be a table with a suitable index. Note that `LOOKUP` can only be used with `INNER` and `LEFT` joins. + +If it is not possible to use the algorithm specified in the hint, an error is signaled. + +### Additional considerations + +- This syntax is consistent with the [SQL Server syntax for join hints](https://docs.microsoft.com/en-us/sql/t-sql/queries/hints-transact-sql-join?view=sql-server-2017), except that: + + - SQL Server uses `LOOP` instead of `LOOKUP`. + + - CockroachDB does not support `LOOP` and instead supports `LOOKUP` for the specific case of nested loop joins with an index. + +- When a join hint is specified, the two tables will not be reordered by the optimizer. The reordering behavior has the following characteristics, which can be affected by hints: + + - Given `a JOIN b`, CockroachDB will not try to commute to `b JOIN a`. This means that you will need to pay attention to this ordering, which is especially important for lookup joins. Without a hint, `a JOIN b` might be executed as `b INNER LOOKUP JOIN a` using an index into `a`, whereas `a INNER LOOKUP JOIN b` requires an index into `b`. + + - `(a JOIN b) JOIN c` might be changed to `a JOIN (b JOIN c)`, but this does not happen if `a JOIN b` uses a hint; the hint forces that particular join to happen as written in the query. + +- Hint usage should be reconsidered with each new release of CockroachDB. Due to improvements in the optimizer, hints specified to work with an older version may cause decreased performance in a newer version. + +## Preferring the nearest index + +Given multiple identical [indexes](indexes.html) that have different locality constraints using [replication zones](configure-replication-zones.html), the optimizer will prefer the index that is closest to the gateway node that is planning the query. In a properly configured geo-distributed cluster, this can lead to performance improvements due to improved data locality and reduced network traffic. + +{{site.data.alerts.callout_info}} +This feature is only available to users with an [enterprise license](enterprise-licensing.html). For insight into how to use this feature to get low latency, consistent reads in multi-region deployments, see the [Duplicate Indexes](topology-follower-reads.html) topology pattern. +{{site.data.alerts.end}} + +This feature enables scenarios such as: + +- Reference data such as a table of postal codes that can be replicated to different regions, and queries will use the copy in the same region. See [Example - zone constraints](#zone-constraints) for more details. +- Optimizing for local reads (potentially at the expense of writes) by adding leaseholder preferences to your zone configuration. See [Example - leaseholder preferences](#leaseholder-preferences) for more details. + +To take advantage of this feature, you need to: + +1. Have an [enterprise license](enterprise-licensing.html). +2. Determine which data consists of reference tables that are rarely updated (such as postal codes) and can therefore be easily replicated to different regions. +3. Create multiple [secondary indexes](indexes.html) on the reference tables. **Note that these indexes must include (in key or using [`STORED`](create-index.html#store-columns)) *every* column that you wish to query**. For example, if you run `SELECT * from db.table` and not every column of `db.table` is in the set of secondary indexes you created, the optimizer will have no choice but to fall back to the primary index. +4. Create [replication zones](configure-replication-zones.html) for each index. + +With the above pieces in place, the optimizer will automatically choose the index nearest the gateway node that is planning the query. + +{{site.data.alerts.callout_info}} +The optimizer does not actually understand geographic locations, i.e., the relative closeness of the gateway node to other nodes that are located to its "east" or "west". It is matching against the [node locality constraints](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes) you provided when you configured your replication zones. +{{site.data.alerts.end}} + +### Examples + +#### Zone constraints + +We can demonstrate the necessary configuration steps using a local cluster. The instructions below assume that you are already familiar with: + +- How to [start a local cluster](start-a-local-cluster.html). +- The syntax for [assigning node locality when configuring replication zones](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes). +- Using [the built-in SQL client](cockroach-sql.html). + +First, start 3 local nodes as shown below. Use the [`--locality`](cockroach-start.html#locality) flag to put them each in a different region as denoted by `region=usa`, `region=eu`, etc. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=usa --insecure --store=/tmp/node0 --listen-addr=localhost:26257 \ + --http-port=8888 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=eu --insecure --store=/tmp/node1 --listen-addr=localhost:26258 \ + --http-port=8889 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=apac --insecure --store=/tmp/node2 --listen-addr=localhost:26259 \ + --http-port=8890 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --insecure --host=localhost --port=26257 +~~~ + +Next, from the SQL client, add your organization name and enterprise license: + +{% include copy-clipboard.html %} +~~~ sh +$ cockroach sql --insecure --host=localhost --port=26257 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING cluster.organization = 'FooCorp - Local Testing'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING enterprise.license = 'xxxxx'; +~~~ + +Create a test database and table. The table will have 3 indexes into the same data. Later, we'll configure the cluster to associate each of these indexes with a different datacenter using replication zones. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE IF NOT EXISTS test; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> USE test; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +CREATE TABLE postal_codes ( + id INT PRIMARY KEY, + code STRING, + INDEX idx_eu (id) STORING (code), + INDEX idx_apac (id) STORING (code) +); +~~~ + +Next, we modify the replication zone configuration via SQL so that: + +- Nodes in the USA will use the primary key index. +- Nodes in the EU will use the `postal_codes@idx_eu` index (which is identical to the primary key index). +- Nodes in APAC will use the `postal_codes@idx_apac` index (which is also identical to the primary key index). + +{% include copy-clipboard.html %} +~~~ sql +ALTER TABLE postal_codes CONFIGURE ZONE USING constraints='["+region=usa"]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +ALTER INDEX postal_codes@idx_eu CONFIGURE ZONE USING constraints='["+region=eu"]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +ALTER INDEX postal_codes@idx_apac CONFIGURE ZONE USING constraints='["+region=apac"]'; +~~~ + +To verify this feature is working as expected, we'll query the database from each of our local nodes as shown below. Each node has been configured to be in a different region, and it should now be using the index pinned to that region. + +{{site.data.alerts.callout_info}} +In a geo-distributed scenario with a cluster that spans multiple datacenters, it may take time for the optimizer to fetch schemas from other nodes the first time a query is planned; thereafter, the schema should be cached locally. + +For example, if you have 11 nodes, you may see 11 queries with high latency due to schema cache misses. Once all nodes have cached the schema locally, the latencies will drop. + +This behavior may also cause the [Statements page of the Web UI](admin-ui-statements-page.html) to show misleadingly high latencies until schemas are cached locally. +{{site.data.alerts.end}} + +As expected, the node in the USA region uses the primary key index. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost --port=26257 --database=test -e 'EXPLAIN SELECT * FROM postal_codes WHERE id=1;' +~~~ + +~~~ + tree | field | description +-------+-------------+----------------------- + | distributed | false + | vectorized | false + scan | | + | table | postal_codes@primary + | spans | /1-/1/# +(5 rows) +~~~ + +As expected, the node in the EU uses the `idx_eu` index. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost --port=26258 --database=test -e 'EXPLAIN SELECT * FROM postal_codes WHERE id=1;' +~~~ + +~~~ + tree | field | description +-------+-------------+---------------------- + | distributed | false + | vectorized | false + scan | | + | table | postal_codes@idx_eu + | spans | /1-/2 +(5 rows) +~~~ + +As expected, the node in APAC uses the `idx_apac` index. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost --port=26259 --database=test -e 'EXPLAIN SELECT * FROM postal_codes WHERE id=1;' +~~~ + +~~~ + tree | field | description +-------+-------------+------------------------ + | distributed | false + | vectorized | false + scan | | + | table | postal_codes@idx_apac + | spans | /1-/2 +(5 rows) +~~~ + +You'll need to make changes to the above configuration to reflect your [production environment](recommended-production-settings.html), but the concepts will be the same. + +#### Leaseholder preferences + +If you provide [leaseholder preferences](configure-replication-zones.html#lease_preferences) in addition to replication zone constraints, the optimizer will attempt to take your leaseholder preferences into account as well when selecting an index for your query. There are several factors to keep in mind: + +- Zone constraints are always respected (hard constraint), whereas lease preferences are taken into account as "additional information" -- as long as they do not contradict the zone constraints. + +- The optimizer does not consider the real-time location of leaseholders when selecting an index; it is pattern matching on the text values passed in the configuration (e.g., the [`ALTER INDEX`](alter-index.html) statements shown below). For the same reason, the optimizer only matches against the first locality in your `lease_preferences` array. + +- The optimizer may use an index that satisfies your leaseholder preferences even though that index has moved to a different node/region due to [leaseholder rebalancing](architecture/replication-layer.html#leaseholder-rebalancing). This can cause slower performance than you expected. Therefore, you should only use this feature if you’re confident you know where the leaseholders will end up based on your cluster's usage patterns. We recommend thoroughly testing your configuration to ensure the optimizer is selecting the index(es) you expect. + +In this example, we'll set up an authentication service using the access token / refresh token pattern from [OAuth 2](https://www.digitalocean.com/community/tutorials/an-introduction-to-oauth-2). To support fast local reads in our geo-distributed use case, we will have 3 indexes into the same authentication data: one for each region of our cluster. We configure each index using zone configurations and lease preferences so that the optimizer will use the local index for better performance. + +The instructions below assume that you are already familiar with: + +- How to [start a local cluster](start-a-local-cluster.html). +- The syntax for [assigning node locality when configuring replication zones](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes). +- Using [the built-in SQL client](cockroach-sql.html). + +First, start 3 local nodes as shown below. Use the [`--locality`](cockroach-start.html#locality) flag to put them each in a different region. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=us-east --insecure --store=/tmp/node0 --listen-addr=localhost:26257 \ + --http-port=8888 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=us-central --insecure --store=/tmp/node1 --listen-addr=localhost:26258 \ + --http-port=8889 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --locality=region=us-west --insecure --store=/tmp/node2 --listen-addr=localhost:26259 \ + --http-port=8890 --join=localhost:26257,localhost:26258,localhost:26259 --background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --insecure --host=localhost --port=26257 +~~~ + +From the SQL client, add your organization name and enterprise license: + +{% include copy-clipboard.html %} +~~~ sh +$ cockroach sql --insecure --host=localhost --port=26257 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING cluster.organization = 'FooCorp - Local Testing'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING enterprise.license = 'xxxxx'; +~~~ + +Create an authentication database and table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE if NOT EXISTS auth; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> USE auth; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE token ( + token_id VARCHAR(100) NULL, + access_token VARCHAR(4000) NULL, + refresh_token VARCHAR(4000) NULL + ); +~~~ + +Create the indexes for each region: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX token_id_west_idx ON token (token_id) STORING (access_token, refresh_token); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX token_id_central_idx ON token (token_id) STORING (access_token, refresh_token); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX token_id_east_idx ON token (token_id) STORING (access_token, refresh_token); +~~~ + +Enter zone configurations to distribute replicas across the cluster as follows: + +- For the "East" index, store 2 replicas in the East, 2 in Central, and 1 in the West. Further, prefer that the leaseholders for that index live in the East or, failing that, in the Central region. +- Follow the same replica and leaseholder patterns for each of the Central and West regions. + +The idea is that, for example, `token_id_east_idx` will have sufficient replicas (2/5) so that even if one replica goes down, the leaseholder will stay in the East region. That way, if a query comes in that accesses the columns covered by that index from the East gateway node, the optimizer will select `token_id_east_idx` for fast reads. + +{{site.data.alerts.callout_info}} +The `ALTER TABLE` statement below is not required since it's later made redundant by the `token_id_west_idx` index. In production, you might go with the `ALTER TABLE` to put your table's lease preferences in the West, and then create only 2 indexes (for East and Central); however, the use of 3 indexes makes the example easier to understand. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE token CONFIGURE ZONE USING + num_replicas = 5, constraints = '{+region=us-east: 1, +region=us-central: 2, +region=us-west: 2}', lease_preferences = '[[+region=us-west], [+region=us-central]]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER INDEX token_id_east_idx CONFIGURE ZONE USING num_replicas = 5, + constraints = '{+region=us-east: 2, +region=us-central: 2, +region=us-west: 1}', lease_preferences = '[[+region=us-east], [+region=us-central]]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER INDEX token_id_central_idx CONFIGURE ZONE USING num_replicas = 5, + constraints = '{+region=us-east: 2, +region=us-central: 2, +region=us-west: 1}', lease_preferences = '[[+region=us-central], [+region=us-east]]'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER INDEX token_id_west_idx CONFIGURE ZONE USING num_replicas = 5, + constraints = '{+region=us-west: 2, +region=us-central: 2, +region=us-east: 1}', lease_preferences = '[[+region=us-west], [+region=us-central]]'; +~~~ + +Next let's [check our zone configurations](show-zone-configurations.html) to make sure they match our expectation: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW ZONE CONFIGURATIONS; +~~~ + +The output should include the following: + +~~~ +TABLE auth.public.token | ALTER TABLE auth.public.token CONFIGURE ZONE USING + | num_replicas = 5, + | constraints = '{+region=us-central: 2, +region=us-east: 1, +region=us-west: 2}', + | lease_preferences = '[[+region=us-west], [+region=us-central]]' +INDEX auth.public.token@token_id_east_idx | ALTER INDEX auth.public.token@token_id_east_idx CONFIGURE ZONE USING + | num_replicas = 5, + | constraints = '{+region=us-central: 2, +region=us-east: 2, +region=us-west: 1}', + | lease_preferences = '[[+region=us-east], [+region=us-central]]' +INDEX auth.public.token@token_id_central_idx | ALTER INDEX auth.public.token@token_id_central_idx CONFIGURE ZONE USING + | num_replicas = 5, + | constraints = '{+region=us-central: 2, +region=us-east: 2, +region=us-west: 1}', + | lease_preferences = '[[+region=us-central], [+region=us-east]]' +INDEX auth.public.token@token_id_west_idx | ALTER INDEX auth.public.token@token_id_west_idx CONFIGURE ZONE USING + | num_replicas = 5, + | constraints = '{+region=us-central: 2, +region=us-east: 1, +region=us-west: 2}', + | lease_preferences = '[[+region=us-west], [+region=us-central]]' +~~~ + +Now that we've set up our indexes the way we want them, we need to insert some data. The first statement below inserts 10,000 rows of dummy data; the second inserts a row with a specific UUID string that we'll later query against to check which index is used. + +{{site.data.alerts.callout_info}} +On a freshly created cluster like this one, you may need to wait a moment after adding the data to give [automatic statistics](#table-statistics) time to update. Then, the optimizer can generate a query plan that uses the expected index. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> INSERT + INTO + token (token_id, access_token, refresh_token) + SELECT + gen_random_uuid()::STRING, + gen_random_uuid()::STRING, + gen_random_uuid()::STRING + FROM + generate_series(1, 10000); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT + INTO + token (token_id, access_token, refresh_token) + VALUES + ( + '2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9', + '49E36152-6152-11E9-8CDC-3682F23211D9', + '4E0E91B6-6152-11E9-BAC1-3782F23211D9' + ); +~~~ + +Finally, we [`EXPLAIN`](explain.html) a [selection query](selection-queries.html) from each node to verify which index is being queried against. For example, when running the query shown below against the `us-west` node, we expect it to use the `token_id_west_idx` index. + +{% include copy-clipboard.html %} +~~~ sh +$ cockroach sql --insecure --host=localhost --port=26259 --database=auth # "West" node +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN + SELECT + access_token, refresh_token + FROM + token + WHERE + token_id = '2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9'; +~~~ + +~~~ + tree | field | description +------------+-------------+-------------------------------------------------------------------------------------------- + | distributed | false + | vectorized | false + render | | + └── scan | | + | table | token@token_id_east_idx + | spans | /"2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9"-/"2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9"/PrefixEnd +(6 rows) +~~~ + +Similarly, queries from the `us-east` node should use the `token_id_east_idx` index (and the same should be true for `us-central`). + +{% include copy-clipboard.html %} +~~~ sh +$ cockroach sql --insecure --host=localhost --port=26257 --database=auth # "East" node +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN + SELECT + access_token, refresh_token + FROM + token + WHERE + token_id = '2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9'; +~~~ + +~~~ + tree | field | description +------------+-------------+-------------------------------------------------------------------------------------------- + | distributed | false + | vectorized | false + render | | + └── scan | | + | table | token@token_id_east_idx + | spans | /"2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9"-/"2E1B5BFE-6152-11E9-B9FD-A7E0F13211D9"/PrefixEnd +(6 rows) +~~~ + +You'll need to make changes to the above configuration to reflect your [production environment](recommended-production-settings.html), but the concepts will be the same. + +## See also + +- [`SET (session variable)`](set-vars.html) +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`RESET CLUSTER SETTING`](reset-cluster-setting.html) +- [`SHOW (session variable)`](show-vars.html) +- [`CREATE STATISTICS`](create-statistics.html) +- [`SHOW STATISTICS`](show-statistics.html) +- [`EXPLAIN`](explain.html) diff --git a/v20.2/create-a-file-server.md b/v20.2/create-a-file-server.md new file mode 100644 index 00000000000..d485cefe24d --- /dev/null +++ b/v20.2/create-a-file-server.md @@ -0,0 +1,125 @@ +--- +title: Create a File Server for Imports and Backups +summary: Learn how to create a simple file server for use with CockroachDB IMPORT and BACKUP +toc: true +--- + +If you need a location to store files for the [`IMPORT`](import.html) process or [CockroachDB enterprise backups](backup.html), but do not have access to (or simply cannot use) cloud storage providers, you can run a local file server. You can then use this file server by leveraging support for our [HTTP Export Storage API](#http-export-storage-api). + +This is especially useful for: + +- Implementing a compatibility layer in front of custom or proprietary storage providers for which CockroachDB does not yet have native support +- Using on-premises storage + +## HTTP export storage API + +CockroachDB tasks that require reading or writing external files (such as [`IMPORT`](import.html) and [`BACKUP`](backup.html)) can use the HTTP Export Storage API by prefacing the address with `http`, e.g., `http://fileserver/mnt/cockroach-exports`. + +This API uses the `GET`, `PUT` and `DELETE` methods. This behaves like you would expect typical HTTP requests to work. After a `PUT` request to some path, a subsequent `GET` request should return the content sent in the `PUT` request body, at least until a `DELETE` request is received for that path. + +## Examples + +You can use any file server software that supports `GET`, `PUT` and `DELETE` methods, but we've included code samples for common ones: + +- [Using PHP with `IMPORT`](#using-php-with-import) +- [Using Python with `IMPORT`](#using-python-with-import) +- [Using Ruby with `IMPORT`](#using-ruby-with-import) +- [Using Caddy as a file server](#using-caddy-as-a-file-server) +- [Using nginx as a file server](#using-nginx-as-a-file-server) + +{{site.data.alerts.callout_info}}We do not recommend using any machines running cockroach as file servers. Using machines that are running cockroach as file servers could negatively impact performance if I/O operations exceed capacity.{{site.data.alerts.end}} + +### Using PHP with `IMPORT` + +The PHP language has an HTTP server built in. You can serve local files using the commands below. For more information about how to import these locally served files, see the documentation for the [`IMPORT`][import] statement. + +{% include copy-clipboard.html %} +~~~ shell +$ cd /path/to/data +$ php -S 127.0.0.1:3000 # files available at e.g., 'http://localhost:3000/data.sql' +~~~ + +### Using Python with `IMPORT` + +The Python language has an HTTP server included in the standard library. You can serve local files using the commands below. For more information about how to import these locally served files, see the documentation for the [`IMPORT`][import] statement. + +{% include copy-clipboard.html %} +~~~ shell +$ cd /path/to/data +$ python -m SimpleHTTPServer 3000 # files available at e.g., 'http://localhost:3000/data.sql' +~~~ + +If you use Python 3, try: + +{% include copy-clipboard.html %} +~~~ shell +$ cd /path/to/data +$ python -m http.server 3000 +~~~ + +### Using Ruby with `IMPORT` + +The Ruby language has an HTTP server included in the standard library. You can serve local files using the commands below. For more information about how to import these locally served files, see the documentation for the [`IMPORT`][import] statement. + +{% include copy-clipboard.html %} +~~~ shell +$ cd /path/to/data +$ ruby -run -ehttpd . -p3000 # files available at e.g., 'http://localhost:3000/data.sql' +~~~ + +### Using Caddy as a file server + +1. [Download the Caddy web server](https://caddyserver.com/download). Before downloading, in the **Customize your build** step, open the list of **Plugins** and make sure to check the `http.upload` option. + +2. Copy the `caddy` binary to the directory containing the files you want to serve, and run it [with an upload directive](https://caddyserver.com/docs/http.upload), either in the command line or via [Caddyfile](https://caddyserver.com/docs/caddyfile). + +- Command line example (with no TLS): + {% include copy-clipboard.html %} + ~~~ shell + $ caddy -root /mnt/cockroach-exports "upload / {" 'to "/mnt/cockroach-exports"' 'yes_without_tls' "}" + ~~~ +- `Caddyfile` example (using a key and cert): + {% include copy-clipboard.html %} + ~~~ shell + tls key cert + root "/mnt/cockroach-exports" + upload / { + to "/mnt/cockroach-exports" + } + ~~~ + +For more information about Caddy, see [its documentation](https://caddyserver.com/docs). + +### Using nginx as a file server + +1. Install `nginx` with the `webdav` module (often included in `-full` or similarly named packages in various distributions). + +2. In the `nginx.conf` file, add a `dav_methods PUT DELETE` directive. For example: + + {% include copy-clipboard.html %} + ~~~ nginx + events { + worker_connections 1024; + } + http { + server { + listen 20150; + location / { + dav_methods PUT DELETE; + root /mnt/cockroach-exports; + sendfile on; + sendfile_max_chunk 1m; + } + } + } + ~~~ + +## See also + +- [`IMPORT`][import] +- [`BACKUP`](backup.html) (*Enterprise only*) +- [`RESTORE`](restore.html) (*Enterprise only*) + + + +[import]: import.html diff --git a/v20.2/create-changefeed.md b/v20.2/create-changefeed.md new file mode 100644 index 00000000000..9559404d184 --- /dev/null +++ b/v20.2/create-changefeed.md @@ -0,0 +1,295 @@ +--- +title: CREATE CHANGEFEED +summary: The CREATE CHANGEFEED statement creates a new enterprise changefeed, which provides row-level change subscriptions in a configurable format to a configurable sink. +toc: true +--- + +{{site.data.alerts.callout_info}} +`CREATE CHANGEFEED` is an [enterprise-only](enterprise-licensing.html) feature. For the core version, see [`EXPERIMENTAL CHANGEFEED FOR`](changefeed-for.html). +{{site.data.alerts.end}} + +The `CREATE CHANGEFEED` [statement](sql-statements.html) creates a new enterprise changefeed, which targets an allowlist of tables, called "watched rows". Every change to a watched row is emitted as a record in a configurable format (`JSON` or Avro) to a configurable sink ([Kafka](https://kafka.apache.org/) or a [cloud storage sink](#cloud-storage-sink)). You can [create](#create-a-changefeed-connected-to-kafka), [pause](#pause-a-changefeed), [resume](#resume-a-paused-changefeed), or [cancel](#cancel-a-changefeed) an enterprise changefeed. + +For more information, see [Change Data Capture](change-data-capture.html). + +## Required privileges + +Changefeeds can only be created by superusers, i.e., [members of the `admin` role](authorization.html#create-and-manage-roles). The admin role exists by default with `root` as the member. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/create_changefeed.html %} +
+ +## Parameters + +Parameter | Description +----------|------------ +`table_name` | The name of the table (or tables in a comma separated list) to create a changefeed for.

**Note:** Changefeeds do not share internal buffers, so each running changefeed will increase total memory usage. To watch multiple tables, we recommend creating a changefeed with a comma-separated list of tables. +`sink` | The location of the configurable sink. The scheme of the URI indicates the type. For more information, see [Sink URI](#sink-uri) below. +`option` / `value` | For a list of available options and their values, see [Options](#options) below. + + + +### Sink URI + +The sink URI follows the basic format of: + +~~~ +'[scheme]://[host]:[port]?[query_parameters]' +~~~ + +URI Component | Description +-------------------+------------------------------------------------------------------ +`scheme` | The type of sink: [`kafka`](#kafka) or any [cloud storage sink](#cloud-storage-sink). +`host` | The sink's hostname or IP address. +`port` | The sink's port. +`query_parameters` | The sink's [query parameters](#query-parameters). + +#### Kafka + +Example of a Kafka sink URI: + +~~~ +'kafka://broker.address.com:9092?topic_prefix=bar_&tls_enabled=true&ca_cert=LS0tLS1CRUdJTiBDRVJUSUZ&sasl_enabled=true&sasl_user=petee&sasl_password=bones' +~~~ + +#### Cloud storage sink + +Use a cloud storage sink to deliver changefeed data to OLAP or big data systems without requiring transport via Kafka. + +{{site.data.alerts.callout_info}} +Currently, cloud storage sinks only work with `JSON` and emits newline-delimited `JSON` files. +{{site.data.alerts.end}} + +Any of the cloud storages below can be used as a sink: + +{{site.data.alerts.callout_info}} +The `scheme` for a cloud storage sink should be prepended with `experimental-`. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/misc/external-urls.md %} + +#### Query parameters + +Query parameters include: + +Parameter |
Sink Type
| Description +-------------------+----------------------------------------------+------------------------------------------------------------------- +`topic_prefix` | [Kafka](#kafka), [cloud](#cloud-storage-sink) | Type: [`STRING`](string.html)

Adds a prefix to all topic names.

For example, `CREATE CHANGEFEED FOR TABLE foo INTO 'kafka://...?topic_prefix=bar_'` would emit rows under the topic `bar_foo` instead of `foo`. +`tls_enabled=true` | [Kafka](#kafka) | Type: [`BOOL`](bool.html)

If `true`, enable Transport Layer Security (TLS) on the connection to Kafka. This can be used with a `ca_cert` (see below). +`ca_cert` | [Kafka](#kafka) | Type: [`STRING`](string.html)

The base64-encoded `ca_cert` file.

Note: To encode your `ca.cert`, run `base64 -w 0 ca.cert`. +`client_cert` | [Kafka](#kafka) | Type: [`STRING`](string.html)

The base64-encoded Privacy Enhanced Mail (PEM) certificate. This is used with `client_key`. +`client_key` | [Kafka](#kafka) | Type: [`STRING`](string.html)

The base64-encoded private key for the PEM certificate. This is used with `client_cert`. +`sasl_enabled` | [Kafka](#kafka) | Type: [`BOOL`](bool.html)

If `true`, [use SASL/PLAIN to authenticate](https://docs.confluent.io/current/kafka/authentication_sasl/authentication_sasl_plain.html). This requires a `sasl_user` and `sasl_password` (see below). +`sasl_user` | [Kafka](#kafka) | Type: [`STRING`](string.html)

Your SASL username. +`sasl_password` | [Kafka](#kafka) | Type: [`STRING`](string.html)

Your SASL password. +`file_size` | [cloud](#cloud-storage-sink) | Type: [`STRING`](string.html)

The file will be flushed (i.e., written to the sink) when it exceeds the specified file size. This can be used with the [`WITH resolved` option](#options), which flushes on a specified cadence.

**Default:** `16MB` + +### Options + +Option | Value | Description +-------|-------|------------ +`updated` | N/A | Include updated timestamps with each row.

If a `cursor` is provided, the "updated" timestamps will match the [MVCC](architecture/storage-layer.html#mvcc) timestamps of the emitted rows, and there is no initial scan. If a `cursor` is not provided, the changefeed will perform an initial scan (as of the time the changefeed was created), and the "updated" timestamp for each change record emitted in the initial scan will be the timestamp of the initial scan. Similarly, when a [backfill is performed for a schema change](change-data-capture.html#schema-changes-with-column-backfill), the "updated" timestamp is set to the first timestamp for when the new schema is valid. +`resolved` | [`INTERVAL`](interval.html) | Periodically emit [resolved timestamps](change-data-capture.html#resolved-def) to the changefeed. Optionally, set a minimum duration between emitting resolved timestamps. If unspecified, all resolved timestamps are emitted.

Example: `resolved='10s'` +`envelope` | `key_only` / `wrapped` | Use `key_only` to emit only the key and no value, which is faster if you only want to know when the key changes.

Default: `envelope=wrapped` +`cursor` | [Timestamp](as-of-system-time.html#parameters) | Emit any changes after the given timestamp, but does not output the current state of the table first. If `cursor` is not specified, the changefeed starts by doing an initial scan of all the watched rows and emits the current value, then moves to emitting any changes that happen after the scan.

When starting a changefeed at a specific `cursor`, the `cursor` cannot be before the configured garbage collection window (see [`gc.ttlseconds`](configure-replication-zones.html#replication-zone-variables)) for the table you're trying to follow; otherwise, the changefeed will error. With default garbage collection settings, this means you cannot create a changefeed that starts more than 25 hours in the past.

`cursor` can be used to [start a new changefeed where a previous changefeed ended.](#start-a-new-changefeed-where-another-ended)

Example: `CURSOR='1536242855577149065.0000000000'` +`format` | `json` / `experimental_avro` | Format of the emitted record. Currently, support for [Avro is limited and experimental](#avro-limitations). For mappings of CockroachDB types to Avro types, [see the table below](#avro-types).

Default: `format=json`. +`confluent_schema_registry` | Schema Registry address | The [Schema Registry](https://docs.confluent.io/current/schema-registry/docs/index.html#sr) address is required to use `experimental_avro`. +`key_in_value` | N/A | Make the [primary key](primary-key.html) of a deleted row recoverable in sinks where each message has a value but not a key (most have a key and value in each message). `key_in_value` is automatically used for these sinks (currently only [cloud storage sinks](#cloud-storage-sink)). +`diff` | N/A | Publish a `before` field with each message, which includes the value of the row before the update was applied. +`compression` | `gzip` | Compress changefeed data files written to a [cloud storage sink](#cloud-storage-sink). Currently, only [Gzip](https://www.gnu.org/software/gzip/) is supported for compression. +`protect_data_from_gc_on_pause` | N/A | When a [changefeed is paused](pause-job.html), ensure that the data needed to [resume the changefeed](resume-job.html) is not garbage collected.

Note: If you use this option, changefeeds left paused can prevent garbage collection for long periods of time. +`schema_change_events` | `default` / `column_changes` | The type of schema change event that triggers the behavior specified by the `schema_change_policy` option:
  • `default`: Include all [`ADD COLUMN`](add-column.html) events for columns that have a non-`NULL` [`DEFAULT` value](default-value.html) or are [computed](computed-columns.html), and all [`DROP COLUMN`](drop-column.html) events.
  • `column_changes`: Include all all schema change events that add or remove any column.

Default: `schema_change_events=default` +`schema_change_policy` | `backfill` / `skip` / `stop` | The behavior to take when an event specified by the `schema_change_events` option occurs:
  • `backfill`: When [schema changes with column backfill](change-data-capture.html#schema-changes-with-column-backfill) are finished, output all watched rows using the new schema.
  • `skip`: Perform no logical backfills.
  • `stop`: Wait for all data preceding the schema change to be resolved before exiting with an error indicating the timestamp at which the schema change occurred.

Default: `schema_change_policy=backfill` +`initial_scan` / `no_initial_scan` | N/A | Control whether or not an initial scan will occur at the start time of a changefeed. `initial_scan` and `no_initial_scan` cannot be used simultaneously. If neither `initial_scan` nor `no_initial_scan` is specified, an initial scan will occur if there is no `cursor`, and will not occur if there is one. This preserves the behavior from previous releases.

Default: `initial_scan`
If used in conjunction with `cursor`, an initial scan will be performed at the cursor timestamp. If no `cursor` is specified, the initial scan is performed at `now()`. + +{{site.data.alerts.callout_info}} + Using the `format=experimental_avro`, `envelope=key_only`, and `updated` options together is rejected. `envelope=key_only` prevents any rows with updated fields from being emitted, which makes the `updated` option meaningless. +{{site.data.alerts.end}} + +#### Avro limitations + +Currently, support for Avro is limited and experimental. Below is a list of unsupported SQL types and values for Avro changefeeds: + +- [Decimals](decimal.html) must have precision specified. +- [Decimals](decimal.html) with `NaN` or infinite values cannot be written in Avro. + + {{site.data.alerts.callout_info}} + To avoid `NaN` or infinite values, add a [`CHECK` constraint](check.html) to prevent these values from being inserted into decimal columns. + {{site.data.alerts.end}} + +- [`TIME`, `DATE`, `INTERVAL`](https://github.com/cockroachdb/cockroach/issues/32472), [`UUID`, `INET`](https://github.com/cockroachdb/cockroach/issues/34417), [`ARRAY`](https://github.com/cockroachdb/cockroach/issues/34420), [`JSONB`](https://github.com/cockroachdb/cockroach/issues/34421), `BIT`, and collated `STRING` are not supported in Avro yet. + +#### Avro types + +Below is a mapping of CockroachDB types to Avro types: + +CockroachDB Type | Avro Type | Avro Logical Type +-----------------+-----------+--------------------- +[`INT`](int.html) | [`LONG`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`BOOL`](bool.html) | [`BOOLEAN`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`FLOAT`](float.html) | [`DOUBLE`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`STRING`](string.html) | [`STRING`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`DATE`](date.html) | [`INT`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | [`DATE`](https://avro.apache.org/docs/1.8.1/spec.html#Date) +[`TIME`](time.html) | [`LONG`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | [`TIME-MICROS`](https://avro.apache.org/docs/1.8.1/spec.html#Time+%28microsecond+precision%29) +[`TIMESTAMP`](timestamp.html) | [`LONG`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | [`TIME-MICROS`](https://avro.apache.org/docs/1.8.1/spec.html#Time+%28microsecond+precision%29) +[`TIMESTAMPTZ`](timestamp.html) | [`LONG`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | [`TIME-MICROS`](https://avro.apache.org/docs/1.8.1/spec.html#Time+%28microsecond+precision%29) +[`DECIMAL`](decimal.html) | [`BYTES`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | [`DECIMAL`](https://avro.apache.org/docs/1.8.1/spec.html#Decimal) +[`UUID`](uuid.html) | [`STRING`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`INET`](inet.html) | [`STRING`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | +[`JSONB`](jsonb.html) | [`STRING`](https://avro.apache.org/docs/1.8.1/spec.html#schema_primitive) | + +## Responses + +The messages (i.e., keys and values) emitted to a Kafka topic are specific to the [`envelope`](#options). The default format is `wrapped`, and the output messages are composed of the following: + +- **Key**: An array always composed of the row's `PRIMARY KEY` field(s) (e.g., `[1]` for `JSON` or `{"id":{"long":1}}` for Avro). +- **Value**: + - One of three possible top-level fields: + - `after`, which contains the state of the row after the update (or `null`' for `DELETE`s). + - `updated`, which contains the updated timestamp. + - `resolved`, which is emitted for records representing resolved timestamps. These records do not include an "after" value since they only function as checkpoints. + - For [`INSERT`](insert.html) and [`UPDATE`](update.html), the current state of the row inserted or updated. + - For [`DELETE`](delete.html), `null`. + +For example: + +Statement | Response +-----------------------------------------------+----------------------------------------------------------------------- +`INSERT INTO office_dogs VALUES (1, 'Petee');` | JSON: `[1] {"after": {"id": 1, "name": "Petee"}}`
Avro: `{"id":{"long":1}} {"after":{"office_dogs":{"id":{"long":1},"name":{"string":"Petee"}}}}` +`DELETE FROM office_dogs WHERE name = 'Petee'` | JSON: `[1] {"after": null}`
Avro: `{"id":{"long":1}} {"after":null}` + +## Examples + +### Create a changefeed connected to Kafka + +{% include copy-clipboard.html %} +~~~ sql +> CREATE CHANGEFEED FOR TABLE name, name2, name3 + INTO 'kafka://host:port' + WITH updated, resolved; +~~~ +~~~ ++--------------------+ +| job_id | ++--------------------+ +| 360645287206223873 | ++--------------------+ +(1 row) +~~~ + +For more information on how to create a changefeed connected to Kafka, see [Change Data Capture](change-data-capture.html#create-a-changefeed-connected-to-kafka). + +### Create a changefeed connected to Kafka using Avro + +{% include copy-clipboard.html %} +~~~ sql +> CREATE CHANGEFEED FOR TABLE name, name2, name3 + INTO 'kafka://host:port' + WITH format = experimental_avro, confluent_schema_registry = ; +~~~ +~~~ ++--------------------+ +| job_id | ++--------------------+ +| 360645287206223873 | ++--------------------+ +(1 row) +~~~ + +For more information on how to create a changefeed that emits an [Avro](https://avro.apache.org/docs/1.8.2/spec.html) record, see [Change Data Capture](change-data-capture.html#create-a-changefeed-connected-to-kafka-using-avro). + +### Create a changefeed connected to a cloud storage sink + +{{site.data.alerts.callout_danger}} +**This is an experimental feature.** The interface and output are subject to change. + +There is an open correctness issue with changefeeds connected to cloud storage sinks where new row information will display with a lower timestamp than what has already been emitted, which violates our [ordering guarantees](change-data-capture.html#ordering-guarantees). +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> CREATE CHANGEFEED FOR TABLE name, name2, name3 + INTO 'experimental-scheme://host?parameters' + WITH updated, resolved; +~~~ +~~~ ++--------------------+ +| job_id | ++--------------------+ +| 360645287206223873 | ++--------------------+ +(1 row) +~~~ + +For more information on how to create a changefeed connected to a cloud storage sink, see [Change Data Capture](change-data-capture.html#create-a-changefeed-connected-to-a-cloud-storage-sink). + +### Manage a changefeed + +Use the following SQL statements to pause, resume, and cancel a changefeed. + +{{site.data.alerts.callout_info}} +Changefeed-specific SQL statements (e.g., `CANCEL CHANGEFEED`) will be added in the future. +{{site.data.alerts.end}} + +#### Pause a changefeed + +{% include copy-clipboard.html %} +~~~ sql +> PAUSE JOB job_id; +~~~ + +For more information, see [`PAUSE JOB`](pause-job.html). + +#### Resume a paused changefeed + +{% include copy-clipboard.html %} +~~~ sql +> RESUME JOB job_id; +~~~ + +For more information, see [`RESUME JOB`](resume-job.html). + +#### Cancel a changefeed + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL JOB job_id; +~~~ + +For more information, see [`CANCEL JOB`](cancel-job.html). + +### Start a new changefeed where another ended + +Find the [high-water timestamp](change-data-capture.html#monitor-a-changefeed) for the ended changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM crdb_internal.jobs WHERE job_id = ; +~~~ +~~~ + job_id | job_type | ... | high_water_timestamp | error | coordinator_id ++--------------------+------------+ ... +--------------------------------+-------+----------------+ + 383870400694353921 | CHANGEFEED | ... | 1537279405671006870.0000000000 | | 1 +(1 row) +~~~ + +Use the `high_water_timestamp` to start the new changefeed: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE CHANGEFEED FOR TABLE name, name2, name3 + INTO 'kafka//host:port' + WITH cursor = ''; +~~~ + +Note that because the cursor is provided, the initial scan is not performed. + +## See also + +- [Change Data Capture](change-data-capture.html) +- [Other SQL Statements](sql-statements.html) +- [Changefeed Dashboard](admin-ui-cdc-dashboard.html) diff --git a/v20.2/create-database.md b/v20.2/create-database.md new file mode 100644 index 00000000000..bd7598561ad --- /dev/null +++ b/v20.2/create-database.md @@ -0,0 +1,97 @@ +--- +title: CREATE DATABASE +summary: The CREATE DATABASE statement creates a new CockroachDB database. +toc: true +--- + +The `CREATE DATABASE` [statement](sql-statements.html) creates a new CockroachDB database. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +Only members of the `admin` role can create new databases. By default, the `root` user belongs to the `admin` role. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/create_database.html %} +
+ +## Parameters + +Parameter | Description +----------|------------ +`IF NOT EXISTS` | Create a new database only if a database of the same name does not already exist; if one does exist, do not return an error. +`name` | The name of the database to create, which [must be unique](#create-fails-name-already-in-use) and follow these [identifier rules](keywords-and-identifiers.html#identifiers). +`encoding` | The `CREATE DATABASE` statement accepts an optional `ENCODING` clause for compatibility with PostgreSQL, but `UTF-8` is the only supported encoding. The aliases `UTF8` and `UNICODE` are also accepted. Values should be enclosed in single quotes and are case-insensitive.

Example: `CREATE DATABASE bank ENCODING = 'UTF-8'`. + +## Example + +### Create a database + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +{% include copy-clipboard.html %} +~~~ +> SHOW DATABASES; +~~~ + +~~~ ++---------------+ +| database_name | ++---------------+ +| bank | +| defaultdb | +| postgres | +| system | ++---------------+ +(4 rows) +~~~ + +### Create fails (name already in use) + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE bank; +~~~ + +~~~ +pq: database "bank" already exists +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE IF NOT EXISTS bank; +~~~ + +SQL does not generate an error, but instead responds `CREATE DATABASE` even though a new database wasn't created. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW DATABASES; +~~~ + +~~~ ++---------------+ +| database_name | ++---------------+ +| bank | +| defaultdb | +| postgres | +| system | ++---------------+ +(4 rows) +~~~ + +## See also + +- [`SHOW DATABASES`](show-databases.html) +- [`RENAME DATABASE`](rename-database.html) +- [`SET DATABASE`](set-vars.html) +- [`DROP DATABASE`](drop-database.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/create-index.md b/v20.2/create-index.md new file mode 100644 index 00000000000..4ebf7c51079 --- /dev/null +++ b/v20.2/create-index.md @@ -0,0 +1,257 @@ +--- +title: CREATE INDEX +summary: The CREATE INDEX statement creates an index for a table. Indexes improve your database's performance by helping SQL quickly locate data. +toc: true +--- + +The `CREATE INDEX` [statement](sql-statements.html) creates an index for a table. [Indexes](indexes.html) improve your database's performance by helping SQL locate data without having to look through every row of a table. + +The following types cannot be included in an index key, but can be stored (and used in a covered query) using the [`STORING` or `COVERING`](create-index.html#store-columns) clause: + +- [`JSONB`](jsonb.html) +- [`ARRAY`](array.html) +- The computed [`TUPLE`](scalar-expressions.html#tuple-constructor) type, even if it is constructed from indexed fields + +To create an index on the schemaless data in a [`JSONB`](jsonb.html) column, use an [inverted index](inverted-indexes.html). + +{{site.data.alerts.callout_info}} +Indexes are automatically created for a table's [`PRIMARY KEY`](primary-key.html) and [`UNIQUE`](unique.html) columns. When querying a table, CockroachDB uses the fastest index. For more information about that process, see [Index Selection in CockroachDB](https://www.cockroachlabs.com/blog/index-selection-cockroachdb-2/). +{{site.data.alerts.end}} + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Synopsis + +**Standard index:** + +
{% include {{ page.version.version }}/sql/diagrams/create_index.html %}
+ +**Inverted index:** + +
{% include {{ page.version.version }}/sql/diagrams/create_inverted_index.html %}
+ +## Parameters + +Parameter | Description +----------|------------ +`UNIQUE` | Apply the [`UNIQUE` constraint](unique.html) to the indexed columns.

This causes the system to check for existing duplicate values on index creation. It also applies the `UNIQUE` constraint at the table level, so the system checks for duplicate values when inserting or updating data. +`INVERTED` | Create an [inverted index](inverted-indexes.html) on the schemaless data in the specified [`JSONB`](jsonb.html) column.

You can also use the PostgreSQL-compatible syntax `USING GIN`. For more details, see [Inverted Indexes](inverted-indexes.html#creation). +`IF NOT EXISTS` | Create a new index only if an index of the same name does not already exist; if one does exist, do not return an error. +`opt_index_name`
`index_name` | The name of the index to create, which must be unique to its table and follow these [identifier rules](keywords-and-identifiers.html#identifiers).

If you do not specify a name, CockroachDB uses the format `__key/idx`. `key` indicates the index applies the `UNIQUE` constraint; `idx` indicates it does not. Example: `accounts_balance_idx` +`table_name` | The name of the table you want to create the index on. +`USING name` | An optional clause for compatibility with third-party tools. Accepted values for `name` are `btree` and `gin`, with `btree` for a standard secondary index and `gin` as the PostgreSQL-compatible syntax for an [inverted index](#create-inverted-indexes) on schemaless data in a `JSONB` column. +`column_name` | The name of the column you want to index. +`ASC` or `DESC`| Sort the column in ascending (`ASC`) or descending (`DESC`) order in the index. How columns are sorted affects query results, particularly when using `LIMIT`.

__Default:__ `ASC` +`STORING ...`| Store (but do not sort) each column whose name you include.

For information on when to use `STORING`, see [Store Columns](#store-columns). Note that columns that are part of a table's [`PRIMARY KEY`](primary-key.html) cannot be specified as `STORING` columns in secondary indexes on the table.

`COVERING` aliases `STORING` and works identically. +`opt_interleave` | You can potentially optimize query performance by [interleaving indexes](interleave-in-parent.html), which changes how CockroachDB stores your data.
{{site.data.alerts.callout_info}}[Hash-sharded indexes](indexes.html#hash-sharded-indexes) cannot be interleaved.{{site.data.alerts.end}} +`opt_partition_by` | An [enterprise-only](enterprise-licensing.html) option that lets you [define index partitions at the row level](partitioning.html). +`USING HASH WITH BUCKET COUNT` | Creates a [hash-sharded index](indexes.html#hash-sharded-indexes) with `n_buckets` number of buckets.
{{site.data.alerts.callout_info}}To enable hash-sharded indexes, set the `experimental_enable_hash_sharded_indexes` [session variable](set-vars.html) to `on`.{{site.data.alerts.end}} +`CONCURRENTLY` | Optional, no-op syntax for PostgreSQL compatibility. All indexes are created concurrently in CockroachDB. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Create standard indexes + +To create the most efficient indexes, we recommend reviewing: + +- [Indexes: Best Practices](indexes.html#best-practices) +- [Index Selection in CockroachDB](https://www.cockroachlabs.com/blog/index-selection-cockroachdb-2) + +#### Single-column indexes + +Single-column indexes sort the values of a single column. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users (name); +~~~ + +Because each query can only use one index, single-column indexes are not typically as useful as multiple-column indexes. + +#### Multiple-column indexes + +Multiple-column indexes sort columns in the order you list them. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users (name, city); +~~~ + +To create the most useful multiple-column indexes, we recommend reviewing our [best practices](indexes.html#indexing-columns). + +#### Unique indexes + +Unique indexes do not allow duplicate values among their columns. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE UNIQUE INDEX ON users (name, id); +~~~ + +This also applies the [`UNIQUE` constraint](unique.html) at the table level, similarly to [`ALTER TABLE`](alter-table.html). The above example is equivalent to: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ADD CONSTRAINT users_name_id_key UNIQUE (name, id); +~~~ + +### Create inverted indexes + +[Inverted indexes](inverted-indexes.html) can be created on schemaless data in a [`JSONB`](jsonb.html) column. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INVERTED INDEX ON promo_codes (rules); +~~~ + +The above example is equivalent to the following PostgreSQL-compatible syntax: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON promo_codes USING GIN (rules); +~~~ + +### Store columns + +Storing a column improves the performance of queries that retrieve (but do not filter) its values. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users (city) STORING (name); +~~~ + +However, to use stored columns, queries must filter another column in the same index. For example, SQL can retrieve `name` values from the above index only when a query's `WHERE` clause filters `city`. + +### Change column sort order + +To sort columns in descending order, you must explicitly set the option when creating the index. (Ascending order is the default.) + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users (city DESC, name); +~~~ + +Note that how a column is ordered in the index will affect the ordering of the index keys, and may affect the efficiency of queries that include an `ORDER BY` clause. + +### Query specific indexes + +Normally, CockroachDB selects the index that it calculates will scan the fewest rows. However, you can override that selection and specify the name of the index you want to use. To find the name, use [`SHOW INDEX`](show-index.html). + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM users; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit ++------------+----------------+------------+--------------+-------------+-----------+---------+----------+ + users | primary | false | 1 | city | ASC | false | false + users | primary | false | 2 | id | ASC | false | false + users | users_name_idx | true | 1 | name | ASC | false | false + users | users_name_idx | true | 2 | city | ASC | false | true + users | users_name_idx | true | 3 | id | ASC | false | true +(5 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT name FROM users@users_name_idx WHERE city='new york'; +~~~ + +~~~ + name ++------------------+ + Catherine Nelson + Devin Jordan + James Hamilton + Judy White + Robert Murphy +(5 rows) +~~~ + +### Create a hash-sharded secondary index + +{% include {{page.version.version}}/performance/use-hash-sharded-indexes.md %} + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE events ( + product_id INT8, + owner UUID, + serial_number VARCHAR, + event_id UUID, + ts TIMESTAMP, + data JSONB, + PRIMARY KEY (product_id, owner, serial_number, ts, event_id) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET experimental_enable_hash_sharded_indexes=on; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON events(ts) USING HASH WITH BUCKET_COUNT=8; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM events; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit +-------------+----------------------------------------+------------+--------------+--------------------------+-----------+---------+----------- + events | primary | false | 1 | product_id | ASC | false | false + events | primary | false | 2 | owner | ASC | false | false + events | primary | false | 3 | serial_number | ASC | false | false + events | primary | false | 4 | ts | ASC | false | false + events | primary | false | 5 | event_id | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 1 | crdb_internal_ts_shard_8 | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 2 | ts | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 3 | product_id | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 4 | owner | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 5 | serial_number | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 6 | event_id | ASC | false | true +(11 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM events; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden +---------------------------+-----------+-------------+----------------+-----------------------------------+--------------------------------------------------+------------ + product_id | INT8 | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + owner | UUID | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + serial_number | VARCHAR | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + event_id | UUID | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + ts | TIMESTAMP | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + data | JSONB | true | NULL | | {} | false + crdb_internal_ts_shard_8 | INT4 | false | NULL | mod(fnv32(CAST(ts AS STRING)), 8) | {events_crdb_internal_ts_shard_8_ts_idx} | true +(7 rows) +~~~ + +## See also + +- [Indexes](indexes.html) +- [`SHOW INDEX`](show-index.html) +- [`DROP INDEX`](drop-index.html) +- [`RENAME INDEX`](rename-index.html) +- [`SHOW JOBS`](show-jobs.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/create-role.md b/v20.2/create-role.md new file mode 100644 index 00000000000..b58bacbcea0 --- /dev/null +++ b/v20.2/create-role.md @@ -0,0 +1,121 @@ +--- +title: CREATE ROLE +summary: The CREATE ROLE statement creates SQL roles, which are groups containing any number of roles and users as members. +toc: true +--- + +The `CREATE ROLE` [statement](sql-statements.html) creates SQL [roles](authorization.html#create-and-manage-roles), which are groups containing any number of roles and users as members. You can assign privileges to roles, and all members of the role (regardless of whether if they are direct or indirect members) will inherit the role's privileges. + +{{site.data.alerts.callout_info}} + CREATE ROLE is no longer an enterprise feature and is now freely available in the core version of CockroachDB. Also, since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `CREATE ROLE` is now an alias for [`CREATE USER`](create-user.html). +{{site.data.alerts.end}} + +## Considerations + +- Role names: + - Are case-insensitive + - Must start with either a letter or underscore + - Must contain only letters, numbers, periods, or underscores + - Must be between 1 and 63 characters. +- After creating roles, you must [grant them privileges to databases and tables](grant.html). +- Roles and users can be members of roles. +- Roles and users share the same namespace and must be unique. +- All privileges of a role are inherited by all of its members. +- There is no limit to the number of members in a role. +- Roles cannot log in. They do not have a password and cannot use certificates. +- Membership loops are not allowed (direct: `A is a member of B is a member of A` or indirect: `A is a member of B is a member of C ... is a member of A`). + +## Required privileges + + To create other roles, the role must have the [`CREATEROLE`](#allow-the-role-to-create-other-roles) parameter set. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/create_role.html %}
+ +## Parameters + +| Parameter | Description | +------------|-------------- +`name` | The name of the role you want to create. Role names are case-insensitive; must start with either a letter or underscore; must contain only letters, numbers, periods, or underscores; and must be between 1 and 63 characters.

Note that roles and [users](create-user.html) share the same namespace and must be unique. +`password` | Let the role [authenticate their access to a secure cluster](authentication.html#client-authentication) using this password. Passwords should be entered as a [string literal](sql-constants.html#string-literals). For compatibility with PostgreSQL, a password can also be entered as an [identifier](#create-a-role-with-a-password-using-an-identifier).

To prevent a role from using [password authentication](authentication.html#client-authentication) and to mandate [certificate-based client authentication](authentication.html#client-authentication), [set the password as `NULL`](#prevent-a-role-from-using-password-authentication). +`VALID UNTIL` | The date and time (in the [`timestamp`](timestamp.html) format) after which the password is not valid. +`LOGIN`/`NOLOGIN` | Allow or disallow a role to login with one of the [client authentication methods](authentication.html#client-authentication).

By default, the parameter is set to `NOLOGIN` for the `CREATE ROLE` statement. +`CREATEROLE`/`NOCREATEROLE` | Allow or disallow the new role to create, alter, and drop other roles.

By default, the parameter is set to `NOCREATEROLE` for all non-admin and non-root users. + +## Examples + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE dev_ops; +~~~ +~~~ +CREATE ROLE 1 +~~~ + +After creating roles, you can [add users to the role](grant-roles.html) and [grant the role privileges](grant.html). + +### Allow the role to create other roles + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE dev with CREATEROLE; +~~~ + +### Create a role with a password using a string literal + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE carl WITH PASSWORD 'ilov3beefjerky'; +~~~ + +~~~ +CREATE ROLE 1 +~~~ + +### Create a role with a password using an identifier + +The following statement sets the password to `ilov3beefjerky`, as above: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE carl WITH PASSWORD ilov3beefjerky; +~~~ + +This is equivalent to the example in the previous section because the password contains only lowercase characters. + +In contrast, the following statement sets the password to `thereisnotomorrow`, even though the password in the syntax contains capitals, because identifiers are normalized automatically: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE carl WITH PASSWORD ThereIsNoTomorrow; +~~~ + +To preserve case in a password specified using identifier syntax, use double quotes: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE carl WITH PASSWORD "ThereIsNoTomorrow"; +~~~ + +### Prevent a role from using password authentication + +The following statement prevents the role from using password authentication and mandates certificate-based client authentication: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE ROLE carl WITH PASSWORD NULL; +~~~ + +## See also + +- [Authorization](authorization.html) +- [`DROP ROLE`](drop-role.html) +- [`GRANT `](grant.html) +- [`REVOKE `](revoke.html) +- [`GRANT `](grant-roles.html) +- [`REVOKE `](revoke-roles.html) +- [`SHOW ROLES`](show-roles.html) +- [`SHOW USERS`](show-users.html) +- [`SHOW GRANTS`](show-grants.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/create-security-certificates-custom-ca.md b/v20.2/create-security-certificates-custom-ca.md new file mode 100644 index 00000000000..2b750795382 --- /dev/null +++ b/v20.2/create-security-certificates-custom-ca.md @@ -0,0 +1,137 @@ +--- +title: Create Security Certificates using a Custom CA +summary: A secure CockroachDB cluster uses TLS for encrypted inter-node and client-node communication. +toc: true +--- + +To secure your CockroachDB cluster's inter-node and client-node communication, you need to provide a Certificate Authority (CA) certificate that has been used to sign keys and certificates (SSLs) for: + +- Nodes +- Clients +- Admin UI (optional) + +To create these certificates and keys, use the `cockroach cert` [commands](cockroach-commands.html) with the appropriate subcommands and flags, use [`openssl` commands](https://wiki.openssl.org/index.php/), or use a [custom CA](create-security-certificates-custom-ca.html) (for example, a public CA or your organizational CA). + + + +This document discusses the following advanced use cases for using security certificates with CockroachDB: + +Approach | Use case description +-------------|------------ +[UI certificate and key](#accessing-the-admin-ui-for-a-secure-cluster) | When you want to access the Admin UI for a secure cluster and avoid clicking through a warning message to get to the UI. +[Split-node certificate](#split-node-certificates) | When your organizational CA requires you to have separate certificates for the node's incoming connections (from SQL and Admin UI clients, and from other CockroachDB nodes) and for outgoing connections to other CockroachDB nodes. +[Split-CA certificates](#split-ca-certificates) | When you have multiple CockroachDB clusters and need to restrict access to clients from accessing the other cluster. + +## Accessing the Admin UI for a secure cluster + +On [accessing the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui) for a secure cluster, your web browser will consider the CockroachDB-issued certificate invalid, because the browser hasn't been configured to trust the CA that issued the certificate. + +For secure clusters, you can avoid getting the warning message by using a certificate issued by a public CA whose certificates are trusted by browsers, in addition to the CockroachDB-created certificates: + +1. Request a certificate from a public CA (for example, [Let's Encrypt](https://letsencrypt.org/)). The certificate must have the IP addresses and DNS names used to reach the Admin UI listed in the `Subject Alternative Name` field. +2. Rename the certificate and key as `ui.crt` and `ui.key`. +3. Add the `ui.crt` and `ui.key` to the [certificate directory](cockroach-cert.html#certificate-directory). `ui.key` must not have group or world permissions (maximum permissions are 0700, or rwx------). You can disable this check by setting the environment variable `COCKROACH_SKIP_KEY_PERMISSION_CHECK=true`. +4. For nodes that are already running, load the `ui.crt` certificate without restarting the node by issuing a `SIGHUP` signal to the cockroach process: + {% include copy-clipboard.html %} + ~~~ shell + pkill -SIGHUP -x cockroach + ~~~ + The `SIGHUP` signal must be sent by the same user running the process (e.g., run with sudo if the cockroach process is running under user root). + +### Node key and certificates + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`node.crt` | Server certificate created using the `cockroach cert` command.

`node.crt` must have `CN=node` and the list of IP addresses and DNS names listed in `Subject Alternative Name` field.

Must be signed by `ca.crt`. +`node.key` | Server key created using the `cockroach cert` command. +`ui.crt` | UI certificate signed by the public CA. `ui.crt` must have the IP addresses and DNS names used to reach the Admin UI listed in `Subject Alternative Name`. +`ui.key` | UI key corresponding to `ui.crt`. + +### Client key and certificates + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate created using the `cockroach cert` command. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`)

Must be signed by `ca.crt`. +`client..key` | Client key created using the `cockroach cert` command. + +## Split node certificates + +The node certificate discussed in the `cockroach cert` command [documentation](cockroach-cert.html) is multifunctional, which means the same certificate is presented for the node's incoming connections (from SQL and Admin UI clients, and from other CockroachDB nodes) and for outgoing connections to other CockroachDB nodes. To make the certificate multi-functional, the `node.crt` created using the `cockroach cert` command has `CN=node` and the list of IP addresses and DNS names listed in `Subject Alternative Name` field. This works if you are also using the CockroachDB CA created using the `cockroach cert` command. However, if you need to use an external public CA or your own organizational CA, the CA policy might not allow it to sign a server certificate containing a CN that is not an IP address or domain name. + +To get around this issue, you can split the node key and certificate into two: + +- `node.crt` and `node.key`: `node.crt` is used as the server certificate when a node receives incoming connections from clients and other nodes. All IP addresses and DNS names for the node must be listed in the `Subject Alternative Name` field. +- `client.node.crt` and `client.node.key`: `client.node.crt` is used as the client certificate when making connections to other nodes. `client.node.crt` must have `CN=node`. + +### Node key and certificates + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate issued by the public CA or your organizational CA. +`node.crt` | Server certificate used when a node receives incoming connections from clients and other nodes.

All IP addresses and DNS names for the node must be listed in `Subject Alternative Name`.

Must be signed by `ca.crt`. +`node.key` | Server key corresponding to `node.crt`. +`client.node.crt` | Client certificate when making connections to other nodes.

Must have `CN=node`.

Must be signed by `ca.crt`. +`client.node.key` | Client key corresponding to `client.node.crt`. + +Optionally, if you have a certificate issued by a public CA to securely access the Admin UI, you need to place the certificate and key (`ui.crt` and `ui.key` respectively) in the directory specified by the `--certs-dir` flag. + +### Client key and certificates + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate issued by the public CA or your organizational CA. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`)

Must be signed by `ca.crt`. +`client..key` | Client key corresponding to `client..crt`. + +## Split CA certificates + +{{site.data.alerts.callout_danger}} +We do not recommend you use split CA certificates unless your organizational security practices mandate you to do so. +{{site.data.alerts.end}} + +If you need to use separate CAs to sign node certificates and client certificates, then you need two CAs and their respective certificates and keys: `ca.crt` and `ca-client.crt`. + +### Node key and certificates + +A node must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate to verify node certificates. +`ca-client.crt` | CA certificate to verify client certificates. +`node.crt` | Server certificate used when a node receives incoming connections from clients and other nodes.

All IP addresses and DNS names for the node must be listed in `Subject Alternative Name`.

Must be signed by `ca.crt`. +`node.key` | Server key corresponding to `node.crt`. +`client.node.crt` | Client certificate when making connections to other nodes. This certificate must be signed using `ca-client.crt`

Must have `CN=node`. +`client.node.key` | Client key corresponding to `client.node.crt`. + +Optionally, if you have a certificate issued by a public CA to securely access the Admin UI, you need to place the certificate and key (`ui.crt` and `ui.key` respectively) in the directory specified by the `--certs-dir` flag. + +### Client key and certificates + +A client must have the following files with file names as specified in the table: + +File name | File usage +-------------|------------ +`ca.crt` | CA certificate. +`client..crt` | Client certificate for `` (e.g., `client.root.crt` for user `root`).

Each `client..crt` must have `CN=` (for example, `CN=marc` for `client.marc.crt`).

Must be signed by `ca-client.crt`. +`client..key` | Client key corresponding to `client..crt`. + +## See also + +- [Manual Deployment](manual-deployment.html): Learn about starting a multi-node secure cluster and accessing it from a client. +- [Start a Node](cockroach-start.html): Learn more about the flags you pass when adding a node to a secure cluster +- [Client Connection Parameters](connection-parameters.html) diff --git a/v20.2/create-security-certificates-openssl.md b/v20.2/create-security-certificates-openssl.md new file mode 100644 index 00000000000..fb4c64e78e2 --- /dev/null +++ b/v20.2/create-security-certificates-openssl.md @@ -0,0 +1,446 @@ +--- +title: Create Security Certificates using Openssl +summary: A secure CockroachDB cluster uses TLS for encrypted inter-node and client-node communication. +toc: true +--- + +To secure your CockroachDB cluster's inter-node and client-node communication, you need to provide a Certificate Authority (CA) certificate that has been used to sign keys and certificates (SSLs) for: + +- Nodes +- Clients +- Admin UI (optional) + +To create these certificates and keys, use the `cockroach cert` [commands](cockroach-commands.html) with the appropriate subcommands and flags, use [`openssl` commands](https://wiki.openssl.org/index.php/), or use a [custom CA](create-security-certificates-custom-ca.html) (for example, a public CA or your organizational CA). + + + +## Subcommands + +Subcommand | Usage +-----------|------ +[`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) | Create an RSA private key. +[`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) | Create CA certificate and CSRs (certificate signing requests). +[`openssl ca`](https://www.openssl.org/docs/manmaster/man1/ca.html) | Create node and client certificates using the CSRs. + +## Configuration files + +To use [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) and [`openssl ca`](https://www.openssl.org/docs/manmaster/man1/ca.html) subcommands, you need the following configuration files: + +File name pattern | File usage +-------------|------------ +`ca.cnf` | CA configuration file +`node.cnf` | Server configuration file +`client.cnf` | Client configuration file + +## Certificate directory + +To create node and client certificates using the OpenSSL commands, you need access to a local copy of the CA certificate and key. We recommend creating all certificates (node, client, and CA certificates), and node and client keys in one place and then distributing them appropriately. Store the CA key somewhere safe and keep a backup; if you lose it, you will not be able to add new nodes or clients to your cluster. + +## Required keys and certificates + +Use the [`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) and [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) subcommands to create all certificates, and node and client keys in a single directory, with the files named as follows: + +### Node key and certificates + +File name pattern | File usage +-------------|------------ +`ca.crt` | CA certificate +`node.crt` | Server certificate +`node.key` | Key for server certificate + +### Client key and certificates + +File name pattern | File usage +-------------|------------ +`ca.crt` | CA certificate. +`client..crt` | Client certificate for `` (for example: `client.root.crt` for user `root`). +`client..key` | Key for the client certificate. + +Note the following: + +- The CA key should not be uploaded to the nodes and clients, so it should be created in a separate directory. + +- Keys (files ending in `.key`) must not have group or world permissions (maximum permissions are 0700, or `rwx------`). This check can be disabled by setting the environment variable `COCKROACH_SKIP_KEY_PERMISSION_CHECK=true`. + +## Examples + +### Step 1. Create the CA key and certificate pair + +1. Create two directories: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs my-safe-directory + ~~~ + - `certs`: Create your CA certificate and all node and client certificates and keys in this directory and then upload the relevant files to the nodes and clients. + - `my-safe-directory`: Create your CA key in this directory and then reference the key when generating node and client certificates. After that, keep the key safe and secret; do not upload it to your nodes or clients. + +2. Create the `ca.cnf` file and copy the following configuration into it. + + You can set the CA certificate expiration period using the `default_days` parameter. We recommend using the CockroachDB default value of the CA certificate expiration period, which is 365 days. + + {% include copy-clipboard.html %} + ~~~ + # OpenSSL CA configuration file + [ ca ] + default_ca = CA_default + + [ CA_default ] + default_days = 365 + database = index.txt + serial = serial.txt + default_md = sha256 + copy_extensions = copy + unique_subject = no + + # Used to create the CA certificate. + [ req ] + prompt=no + distinguished_name = distinguished_name + x509_extensions = extensions + + [ distinguished_name ] + organizationName = Cockroach + commonName = Cockroach CA + + [ extensions ] + keyUsage = critical,digitalSignature,nonRepudiation,keyEncipherment,keyCertSign + basicConstraints = critical,CA:true,pathlen:1 + + # Common policy for nodes and users. + [ signing_policy ] + organizationName = supplied + commonName = optional + + # Used to sign node certificates. + [ signing_node_req ] + keyUsage = critical,digitalSignature,keyEncipherment + extendedKeyUsage = serverAuth,clientAuth + + # Used to sign client certificates. + [ signing_client_req ] + keyUsage = critical,digitalSignature,keyEncipherment + extendedKeyUsage = clientAuth + ~~~ + + {{site.data.alerts.callout_danger}} + The `keyUsage` and `extendedkeyUsage` parameters are vital for CockroachDB functions. You can modify or omit other parameters as per your preferred OpenSSL configuration and you can add additional usages, but do not omit `keyUsage` and `extendedkeyUsage` parameters or remove the listed usages. + {{site.data.alerts.end}} + +3. Create the CA key using the [`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl genrsa -out my-safe-directory/ca.key 2048 + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ chmod 400 my-safe-directory/ca.key + ~~~ + +4. Create the CA certificate using the [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl req \ + -new \ + -x509 \ + -config ca.cnf \ + -key my-safe-directory/ca.key \ + -out certs/ca.crt \ + -days 365 \ + -batch + ~~~ + +6. Reset database and index files: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm -f index.txt serial.txt + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ touch index.txt + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ echo '01' > serial.txt + ~~~ + +### Step 2. Create the certificate and key pairs for nodes + +In the following steps, replace the placeholder text in the code with the actual username and node address. + +1. Create the `node.cnf` file for the first node and copy the following configuration into it: + + {% include copy-clipboard.html %} + ~~~ + # OpenSSL node configuration file + [ req ] + prompt=no + distinguished_name = distinguished_name + req_extensions = extensions + + [ distinguished_name ] + organizationName = Cockroach + + [ extensions ] + subjectAltName = critical,DNS:,DNS:,IP: + ~~~ + + {{site.data.alerts.callout_danger}} + The `subjectAltName` parameter is vital for CockroachDB functions. You can modify or omit other parameters as per your preferred OpenSSL configuration, but do not omit the `subjectAltName` parameter. + {{site.data.alerts.end}} + +2. Create the key for the first node using the [`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl genrsa -out certs/node.key 2048 + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ chmod 400 certs/node.key + ~~~ + +3. Create the CSR for the first node using the [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl req \ + -new \ + -config node.cnf \ + -key certs/node.key \ + -out node.csr \ + -batch + ~~~ + +4. Sign the node CSR to create the node certificate for the first node using the [`openssl ca`](https://www.openssl.org/docs/manmaster/man1/ca.html) command. + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl ca \ + -config ca.cnf \ + -keyfile my-safe-directory/ca.key \ + -cert certs/ca.crt \ + -policy signing_policy \ + -extensions signing_node_req \ + -out certs/node.crt \ + -outdir certs/ \ + -in node.csr \ + -batch + ~~~ + +5. Verify the values in the `Subject Alternative Name` field in the certificate: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl x509 -in certs/node.crt -text | grep "X509v3 Subject Alternative Name" -A 1 + ~~~ + + Sample output: + + ~~~ + X509v3 Subject Alternative Name: critical + DNS:localhost, DNS:node.example.io, IP Address:127.0.0.1 + ~~~ + +### Step 3. Create the certificate and key pair for the first user + +In the following steps, replace the placeholder text in the code with the actual username. + +1. Create the `client.cnf` file for the first user and copy the following configuration into it: + + {% include copy-clipboard.html %} + ~~~ + [ req ] + prompt=no + distinguished_name = distinguished_name + req_extensions = extensions + + [ distinguished_name ] + organizationName = Cockroach + commonName = + + [ extensions ] + subjectAltName = DNS:root + ~~~ + + {{site.data.alerts.callout_danger}} + The `commonName` and `subjectAltName` parameters are vital for CockroachDB functions. You can modify or omit other parameters as per your preferred OpenSSL configuration, but do not omit the `commonName` parameter or modify the `subjectAltName` parameter. + {{site.data.alerts.end}} + +2. Create the key for the first client using the [`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl genrsa -out certs/client..key 2048 + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ chmod 400 certs/client..key + ~~~ + +3. Create the CSR for the first client using the [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl req \ + -new \ + -config client.cnf \ + -key certs/client..key \ + -out client..csr \ + -batch + ~~~ + +4. Sign the client CSR to create the client certificate for the first client using the [`openssl ca`](https://www.openssl.org/docs/manmaster/man1/ca.html) command. + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl ca \ + -config ca.cnf \ + -keyfile my-safe-directory/ca.key \ + -cert certs/ca.crt \ + -policy signing_policy \ + -extensions signing_client_req \ + -out certs/client..crt \ + -outdir certs/ \ + -in client..csr \ + -batch + ~~~ + +5. Verify the values in the `CN` field in the certificate: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl x509 -in certs/client..crt -text | grep CN= + ~~~ + + Sample Output: + + ~~~ + Issuer: O=Cockroach, CN=Cockroach CA + Subject: O=Cockroach, CN=maxroach + ~~~ + +### Step 4. Start a local cluster and connect using a connection URL + +1. Start a single-node cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node --certs-dir=certs --cert-principal-map=:node,:root --background + ~~~ + +2. Connect to the cluster using a connection URL: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --url='postgres://:26257/?sslmode=verify-full&sslrootcert=certs/ca.crt&sslcert=certs/client..crt&sslkey=certs/client..key&sslmode=verify-full' + ~~~ + +3. Create a new SQL user: + + {% include copy-clipboard.html %} + ~~~ sql + > create user ; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +### Step 5. Create the certificate and key pair for a client + +In the following steps, replace the placeholder text in the code with the actual username. + +1. Edit the `client.cnf` file for the client and copy the following configuration into it: + + {% include copy-clipboard.html %} + ~~~ + [ req ] + prompt=no + distinguished_name = distinguished_name + + [ distinguished_name ] + organizationName = Cockroach + commonName = + ~~~ + + {{site.data.alerts.callout_danger}} + The `commonName` parameter is vital for CockroachDB functions. You can modify or omit other parameters as per your preferred OpenSSL configuration, but do not omit the `commonName` parameter. {{site.data.alerts.end}} + +2. Create the key for the first client using the [`openssl genrsa`](https://www.openssl.org/docs/manmaster/man1/genrsa.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl genrsa -out certs/client..key 2048 + ~~~ + {% include copy-clipboard.html %} + ~~~ shell + $ chmod 400 certs/client..key + ~~~ + +3. Create the CSR for the first client using the [`openssl req`](https://www.openssl.org/docs/manmaster/man1/req.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl req \ + -new \ + -config client.cnf \ + -key certs/client..key \ + -out client..csr \ + -batch + ~~~ + +4. Sign the client CSR to create the client certificate for the first client using the [`openssl ca`](https://www.openssl.org/docs/manmaster/man1/ca.html) command. + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl ca \ + -config ca.cnf \ + -keyfile my-safe-directory/ca.key \ + -cert certs/ca.crt \ + -policy signing_policy \ + -extensions signing_client_req \ + -out certs/client..crt \ + -outdir certs/ \ + -in client..csr \ + -batch + ~~~ + +5. Verify the values in the `CN` field in the certificate: + + {% include copy-clipboard.html %} + ~~~ shell + $ openssl x509 -in certs/client..crt -text | grep CN= + ~~~ + + Sample output: + + ~~~ + Issuer: O=Cockroach, CN=Cockroach CA + Subject: O=Cockroach, CN=roach + ~~~ + +6. Connect to the SQL client using the client certificate: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --url='postgres://@:26257/?sslmode=verify-full&sslrootcert=certs/ca.crt&sslcert=certs/client..crt&sslkey=certs/client..key&sslmode=verify-full' + ~~~ + +For each node in your deployment, repeat [Step 2](#step-2-create-the-certificate-and-key-pairs-for-nodes) and upload the CA certificate and node key and certificate to the node. For each client, repeat [Step 5](#step-5-create-the-certificate-and-key-pair-for-a-client) and upload the CA certificate and client key and certificate to the client. + +After you have uploaded all the keys and certificates to the corresponding nodes and clients, remove the `.pem` files in the `certs` directory. These files are unnecessary duplicates of the `.crt` files that CockroachDB requires. + +## See also + +- [Manual Deployment](manual-deployment.html): Learn about starting a multi-node secure cluster and accessing it from a client. +- [Start a Node](cockroach-start.html): Learn more about the flags you pass when adding a node to a secure cluster +- [Client Connection Parameters](connection-parameters.html) diff --git a/v20.2/create-sequence.md b/v20.2/create-sequence.md new file mode 100644 index 00000000000..541bf5fe761 --- /dev/null +++ b/v20.2/create-sequence.md @@ -0,0 +1,208 @@ +--- +title: CREATE SEQUENCE +summary: +toc: true +--- + +The `CREATE SEQUENCE` [statement](sql-statements.html) creates a new sequence in a database. Use a sequence to auto-increment integers in a table. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Considerations + +- Using a sequence is slower than [auto-generating unique IDs with the `gen_random_uuid()`, `uuid_v4()` or `unique_rowid()` built-in functions](sql-faqs.html#how-do-i-auto-generate-unique-row-ids-in-cockroachdb). Incrementing a sequence requires a write to persistent storage, whereas auto-generating a unique ID does not. Therefore, use auto-generated unique IDs unless an incremental sequence is preferred or required. +- A column that uses a sequence can have a gap in the sequence values if a transaction advances the sequence and is then rolled back. Sequence updates are committed immediately and aren't rolled back along with their containing transaction. This is done to avoid blocking concurrent transactions that use the same sequence. +- {% include {{page.version.version}}/performance/use-hash-sharded-indexes.md %} +- If a table references a sequence, and the reference explicitly specifies a database name, that [database cannot be renamed](rename-database.html). In this case, you can drop the column in the table that references the sequence, or you can modify the reference so that it does not specify the database name. + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/create_sequence.html %}
+ +## Parameters + + + + Parameter | Description +-----------|------------ +`seq_name` | The name of the sequence to be created, which must be unique within its database and follow the [identifier rules](keywords-and-identifiers.html#identifiers). When the parent database is not set as the default, the name must be formatted as `database.seq_name`. +`INCREMENT` | The value by which the sequence is incremented. A negative number creates a descending sequence. A positive number creates an ascending sequence.

**Default:** `1` +`MINVALUE` | The minimum value of the sequence. Default values apply if not specified or if you enter `NO MINVALUE`.

**Default for ascending:** `1`

**Default for descending:** `MININT` +`MAXVALUE` | The maximum value of the sequence. Default values apply if not specified or if you enter `NO MAXVALUE`.

**Default for ascending:** `MAXINT`

**Default for descending:** `-1` +`START` | The first value of the sequence.

**Default for ascending:** `1`

**Default for descending:** `-1` +`NO CYCLE` | Currently, all sequences are set to `NO CYCLE` and the sequence will not wrap. +`OWNED BY column_name` | Associates the sequence to a particular column. If that column or its parent table is dropped, the sequence will also be dropped.
Specifying an owner column with `OWNED BY` replaces any existing owner column on the sequence. To remove existing column ownership on the sequence and make the column free-standing, specify `OWNED BY NONE`.

**Default:** `NONE` +`opt_temp` | Defines the sequence as a session-scoped temporary sequence. For more information, see [Temporary sequences](#temporary-sequences).

**Support for temporary sequences is [experimental](experimental-features.html#temporary-objects)**. + + + +## Sequence functions + +We support the following [SQL sequence functions](functions-and-operators.html): + +- `nextval('seq_name')` +- `currval('seq_name')` +- `lastval()` +- `setval('seq_name', value, is_called)` + +## Temporary sequences + + CockroachDB supports session-scoped temporary sequences. Unlike persistent sequences, temporary sequences can only be accessed from the session in which they were created, and they are dropped at the end of the session. You can create temporary sequences on both persistent tables and [temporary tables](temporary-tables.html). + +{{site.data.alerts.callout_danger}} +**This is an experimental feature**. The interface and output are subject to change. For details, see the tracking issue [cockroachdb/cockroach#46260](https://github.com/cockroachdb/cockroach/issues/46260). +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +Temporary tables must be enabled in order to use temporary sequences. By default, temporary tables are disabled in CockroachDB. To enable temporary tables, set the `experimental_enable_temp_tables` [session variable](set-vars.html) to `on`. +{{site.data.alerts.end}} + +### Details + +- Temporary sequences are automatically dropped at the end of the session. +- A temporary sequence can only be accessed from the session in which it was created. +- Temporary sequences persist across transactions in the same session. +- Temporary sequences cannot be converted to persistent sequences. + +{{site.data.alerts.callout_info}} +Like [temporary tables](temporary-tables.html), temporary sequences are not in the `public` schema. Instead, when you create the first temporary table, view, or sequence for a session, CockroachDB generates a single temporary schema (`pg_temp_`) for all of the temporary objects in the current session for a database. +{{site.data.alerts.end}} + +### Usage + +To create a temporary sequence, add [`TEMP`/`TEMPORARY`](sql-grammar.html#opt_temp) to a `CREATE SEQUENCE` statement. + +For example: + +{% include copy-clipboard.html %} +~~~ sql +> SET experimental_enable_temp_tables=on; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TEMP SEQUENCE temp_seq START 1 INCREMENT 1; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE temp_seq; +~~~ + +~~~ + table_name | create_statement +-------------+-------------------------------------------------------------------------------------------- + temp_seq | CREATE TEMP SEQUENCE temp_seq MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1 +(1 row) +~~~ + +## Examples + +### Create a sequence with default settings + +In this example, we create a sequence with default settings. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE SEQUENCE customer_seq; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE customer_seq; +~~~ + +~~~ + table_name | create_statement +---------------+------------------------------------------------------------------------------------------- + customer_seq | CREATE SEQUENCE customer_seq MINVALUE 1 MAXVALUE 9223372036854775807 INCREMENT 1 START 1 +(1 row) +~~~ + +### Create a sequence with user-defined settings + +In this example, we create a sequence that starts at -1 and descends in increments of 2. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE SEQUENCE desc_customer_list START -1 INCREMENT -2; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE desc_customer_list; +~~~ + +~~~ + table_name | create_statement +---------------------+----------------------------------------------------------------------------------------------------- + desc_customer_list | CREATE SEQUENCE desc_customer_list MINVALUE -9223372036854775808 MAXVALUE -1 INCREMENT -2 START -1 +(1 row) +~~~ + +### View the current value of a sequence + +To view the current value without incrementing the sequence, use: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customer_seq; +~~~ + +~~~ + last_value | log_cnt | is_called +-------------+---------+------------ + 3 | 0 | true +(1 row) +~~~ + +{{site.data.alerts.callout_info}}The log_cnt and is_called columns are returned only for PostgreSQL compatibility; they are not stored in the database.{{site.data.alerts.end}} + +If a value has been obtained from the sequence in the current session, you can also use the `currval('seq_name')` function to get that most recently obtained value: + +~~~ sql +> SELECT currval('customer_seq'); +~~~ + +~~~ + currval +----------- + 3 +(1 row) +~~~ + +### List all sequences + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.sequences; +~~~ + +~~~ + sequence_catalog | sequence_schema | sequence_name | data_type | numeric_precision | numeric_precision_radix | numeric_scale | start_value | minimum_value | maximum_value | increment | cycle_option +-------------------+-------------------------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------- + movr | pg_temp_1585153897131110000_1 | temp_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO + movr | public | customer_seq | bigint | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO + movr | public | desc_customer_list | bigint | 64 | 2 | 0 | -1 | -9223372036854775808 | -1 | -2 | NO +(3 rows) +~~~ + +## See also + +- [`ALTER SEQUENCE`](alter-sequence.html) +- [`RENAME SEQUENCE`](rename-sequence.html) +- [`DROP SEQUENCE`](drop-sequence.html) +- [`SHOW CREATE`](show-create.html) +- [`SHOW SEQUENCES`](show-sequences.html) +- [Functions and Operators](functions-and-operators.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/create-statistics.md b/v20.2/create-statistics.md new file mode 100644 index 00000000000..2ad19b0e6bf --- /dev/null +++ b/v20.2/create-statistics.md @@ -0,0 +1,127 @@ +--- +title: CREATE STATISTICS +summary: Use the CREATE STATISTICS statement to generate table statistics for the cost-based optimizer to use. +toc: true +--- +Use the `CREATE STATISTICS` [statement](sql-statements.html) to generate table statistics for the [cost-based optimizer](cost-based-optimizer.html) to use. + +Once you [create a table](create-table.html) and load data into it (e.g., [`INSERT`](insert.html), [`IMPORT`](import.html)), table statistics can be generated. Table statistics help the cost-based optimizer determine the cardinality of the rows used in each query, which helps to predict more accurate costs. + +`CREATE STATISTICS` automatically figures out which columns to get statistics on — specifically, it chooses: + +- Columns that are part of the primary key or an index (in other words, all indexed columns). +- Up to 100 non-indexed columns (unless you specify which columns to create statistics on, as shown in [this example](#create-statistics-on-a-specific-column)). + +{{site.data.alerts.callout_info}} +[Automatic statistics is enabled by default](cost-based-optimizer.html#table-statistics); most users don't need to issue `CREATE STATISTICS` statements directly. +{{site.data.alerts.end}} + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/create_stats.html %} +
+ +## Required Privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database. + +## Parameters + +| Parameter | Description | +|-----------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `statistics_name` | The name of the set of statistics you are creating. | +| `opt_stats_columns` | The name of the column(s) you want to create statistics for. | +| `create_stats_target` | The name of the table you want to create statistics for. | +| `opt_as_of_clause` | Used to create historical stats using the [`AS OF SYSTEM TIME`](as-of-system-time.html) clause. For instructions, see [Create statistics as of a given time](#create-statistics-as-of-a-given-time). | + +## Examples + +### Create statistics on a specific column + +{% include copy-clipboard.html %} +~~~ sql +> CREATE STATISTICS students ON id FROM students_by_list; +~~~ + +{{site.data.alerts.callout_info}} +Multi-column statistics are not supported yet. +{{site.data.alerts.end}} + +### Create statistics on a default set of columns + +The `CREATE STATISTICS` statement shown below automatically figures out which columns to get statistics on — specifically, it chooses: + +- Columns that are part of the primary key or an index (in other words, all indexed columns). +- Up to 100 non-indexed columns. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE STATISTICS students FROM students_by_list; +~~~ + +### Create statistics as of a given time + +To create statistics as of a given time (in this example, 1 minute ago to avoid interfering with the production workload), run a statement like the following: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE STATISTICS employee_stats FROM employees AS OF SYSTEM TIME '-1m'; +~~~ + +For more information about how the `AS OF SYSTEM TIME` clause works, including supported time formats, see [`AS OF SYSTEM TIME`](as-of-system-time.html). + +### Delete statistics + +{% include {{ page.version.version }}/misc/delete-statistics.md %} + +### View statistics jobs + +Every time the `CREATE STATISTICS` statement is executed, it kicks off a background job. This is true for queries issued by your application as well as queries issued by the [automatic stats feature](cost-based-optimizer.html#table-statistics). + +To view statistics jobs, there are two options: + +1. Use [`SHOW JOBS`](show-jobs.html) to see all statistics jobs that were created by user queries (i.e., someone entering `CREATE STATISTICS` at the SQL prompt or via application code): + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM [SHOW JOBS] WHERE job_type LIKE '%CREATE STATS%'; + ~~~ + + ~~~ + job_id | job_type | description | statement | user_name | status | running_status | created | started | finished | modified | fraction_completed | error | coordinator_id + --------------------+--------------+------------------------------------------------------------------+-----------+-----------+-----------+----------------+----------------------------+----------------------------+----------------------------+----------------------------+--------------------+-------+---------------- + 441281249412743169 | CREATE STATS | CREATE STATISTICS salary_stats FROM employees.public.salaries | | root | succeeded | | 2019-04-08 15:52:30.040531 | 2019-04-08 15:52:30.046646 | 2019-04-08 15:52:32.757519 | 2019-04-08 15:52:32.757519 | 1 | | 1 + 441281163978637313 | CREATE STATS | CREATE STATISTICS employee_stats FROM employees.public.employees | | root | succeeded | | 2019-04-08 15:52:03.968099 | 2019-04-08 15:52:03.972557 | 2019-04-08 15:52:05.168809 | 2019-04-08 15:52:05.168809 | 1 | | 1 + (2 rows) + ~~~ + +2. Use `SHOW AUTOMATIC JOBS` to see statistics jobs that were created by the [automatic statistics feature](cost-based-optimizer.html#table-statistics): + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM [SHOW AUTOMATIC JOBS] WHERE job_type LIKE '%CREATE STATS%'; + ~~~ + + ~~~ + job_id | job_type | description | statement | user_name | status | running_status | created | started | finished | modified | fraction_completed | error | coordinator_id + --------------------+-------------------+------------------------------------------------------------+-------------------------------------------------------------------------------------------+-----------+-----------+----------------+----------------------------+----------------------------+----------------------------+----------------------------+--------------------+-------+---------------- + 441280366254850049 | AUTO CREATE STATS | Table statistics refresh for employees.public.departments | CREATE STATISTICS __auto__ FROM [55] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:48:00.522119 | 2019-04-08 15:48:00.52663 | 2019-04-08 15:48:00.541608 | 2019-04-08 15:48:00.541608 | 1 | | 1 + 441280364809289729 | AUTO CREATE STATS | Table statistics refresh for employees.public.titles | CREATE STATISTICS __auto__ FROM [60] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:48:00.080971 | 2019-04-08 15:48:00.083117 | 2019-04-08 15:48:00.515766 | 2019-04-08 15:48:00.515767 | 1 | | 1 + 441280356286201857 | AUTO CREATE STATS | Table statistics refresh for employees.public.salaries | CREATE STATISTICS __auto__ FROM [59] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:47:57.479929 | 2019-04-08 15:47:57.482235 | 2019-04-08 15:48:00.075025 | 2019-04-08 15:48:00.075025 | 1 | | 1 + 441280352161693697 | AUTO CREATE STATS | Table statistics refresh for employees.public.employees | CREATE STATISTICS __auto__ FROM [58] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:47:56.221223 | 2019-04-08 15:47:56.223664 | 2019-04-08 15:47:57.474159 | 2019-04-08 15:47:57.474159 | 1 | | 1 + 441280352070434817 | AUTO CREATE STATS | Table statistics refresh for employees.public.dept_manager | CREATE STATISTICS __auto__ FROM [57] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:47:56.193375 | 2019-04-08 15:47:56.195813 | 2019-04-08 15:47:56.215114 | 2019-04-08 15:47:56.215114 | 1 | | 1 + 441280350791401473 | AUTO CREATE STATS | Table statistics refresh for employees.public.dept_emp | CREATE STATISTICS __auto__ FROM [56] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:47:55.803052 | 2019-04-08 15:47:55.806071 | 2019-04-08 15:47:56.187153 | 2019-04-08 15:47:56.187154 | 1 | | 1 + 441279760786096129 | AUTO CREATE STATS | Table statistics refresh for test.public.kv | CREATE STATISTICS __auto__ FROM [53] WITH OPTIONS THROTTLING 0.9 AS OF SYSTEM TIME '-30s' | root | succeeded | | 2019-04-08 15:44:55.747725 | 2019-04-08 15:44:55.754582 | 2019-04-08 15:44:55.775664 | 2019-04-08 15:44:55.775665 | 1 | | 1 + (7 rows) + ~~~ + +## See Also + +- [Cost-Based Optimizer](cost-based-optimizer.html) +- [`SHOW STATISTICS`](show-statistics.html) +- [`CREATE TABLE`](create-table.html) +- [`INSERT`](insert.html) +- [`IMPORT`](import.html) +- [`SHOW JOBS`](show-jobs.html) +- [SQL Statements](sql-statements.html) diff --git a/v20.2/create-table-as.md b/v20.2/create-table-as.md new file mode 100644 index 00000000000..9d64bb7143f --- /dev/null +++ b/v20.2/create-table-as.md @@ -0,0 +1,353 @@ +--- +title: CREATE TABLE AS +summary: The CREATE TABLE AS statement persists the result of a query into the database for later reuse. +toc: true +--- + +The `CREATE TABLE ... AS` [statement](sql-statements.html) creates a new table from a [selection query](selection-queries.html). + + +## Intended use + +Tables created with `CREATE TABLE ... AS` are intended to persist the +result of a query for later reuse. + +This can be more efficient than a [view](create-view.html) when the +following two conditions are met: + +- The result of the query is used as-is multiple times. +- The copy needs not be kept up-to-date with the original table over time. + +When the results of a query are reused multiple times within a larger +query, a view is advisable instead. The query optimizer can "peek" +into the view and optimize the surrounding query using the primary key +and indices of the tables mentioned in the view query. + +A view is also advisable when the results must be up-to-date; a view +always retrieves the current data from the tables that the view query +mentions. + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database. + +## Synopsis +
+ + +

+ +
+{% include {{ page.version.version }}/sql/diagrams/create_table_as.html %} +
+ +
+ +
+ {% include {{ page.version.version }}/sql/diagrams/create_table_as.html %} +
+ +**create_as_col_qual_list ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/create_as_col_qual_list.html %} +
+ +**create_as_constraint_def ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/create_as_constraint_def.html %} +
+ +
+ +## Parameters + + + + Parameter | Description +-----------|------------- + `IF NOT EXISTS` | Create a new table only if a table of the same name does not already exist in the database; if one does exist, do not return an error.

Note that `IF NOT EXISTS` checks the table name only; it does not check if an existing table has the same columns, indexes, constraints, etc., of the new table. + `table_name` | The name of the table to create, which must be unique within its database and follow these [identifier rules](keywords-and-identifiers.html#identifiers). When the parent database is not set as the default, the name must be formatted as `database.name`.

The [`UPSERT`](upsert.html) and [`INSERT ON CONFLICT`](insert.html) statements use a temporary table called `excluded` to handle uniqueness conflicts during execution. It's therefore not recommended to use the name `excluded` for any of your tables. + `column_name` | The name of the column you want to use instead of the name of the column from `select_stmt`. + `create_as_col_qual_list` | An optional column definition, which may include [primary key constraints](primary-key.html) and [column family assignments](column-families.html). + `family_def` | An optional [column family definition](column-families.html). Column family names must be unique within the table but can have the same name as columns, constraints, or indexes. + `create_as_constraint_def` | An optional [primary key constraint](primary-key.html). + `select_stmt` | A [selection query](selection-queries.html) to provide the data. + `opt_temp_create_table` | Defines the table as a session-scoped temporary table. For more information, see [Temporary Tables](temporary-tables.html).
**Support for temporary tables is experimental**. The interface and output are subject to change. + +## Limitations + +Tables created with `CREATE TABLE ... AS` are not [interleaved](interleave-in-parent.html) with other tables. +The default rules for [column families](column-families.html) apply. + +The [primary key](primary-key.html) of tables created with `CREATE TABLE ... AS` is not automatically derived from the query results. You must specify new primary keys at table creation. For examples, see [Specify a primary key](create-table-as.html#specify-a-primary-key) and [Specify a primary key for partitioning](create-table-as.html#specify-a-primary-key-for-partitioning). + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Create a table from a `SELECT` query + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users WHERE city = 'new york'; +~~~ +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 + 19999999-9999-4a00-8000-000000000005 | new york | Nicole Mcmahon | 11540 Patton Extensions | 0303726947 +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny AS SELECT * FROM users WHERE city = 'new york'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny; +~~~ +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 + 19999999-9999-4a00-8000-000000000005 | new york | Nicole Mcmahon | 11540 Patton Extensions | 0303726947 +(6 rows) +~~~ + +### Change column names + +This statement creates a copy of an existing table but with changed column names: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny_names (user_id, user_name) AS SELECT id, name FROM users WHERE city = 'new york'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny_names; +~~~ +~~~ + user_id | user_name ++--------------------------------------+------------------+ + 00000000-0000-4000-8000-000000000000 | Robert Murphy + 051eb851-eb85-4ec0-8000-000000000001 | James Hamilton + 0a3d70a3-d70a-4d80-8000-000000000002 | Judy White + 0f5c28f5-c28f-4c00-8000-000000000003 | Devin Jordan + 147ae147-ae14-4b00-8000-000000000004 | Catherine Nelson + 19999999-9999-4a00-8000-000000000005 | Nicole Mcmahon +(6 rows) +~~~ + +### Create a table from a `VALUES` clause + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE drivers (id, city, name) AS VALUES (gen_random_uuid(), 'new york', 'Harry Potter'), (gen_random_uuid(), 'seattle', 'Evelyn Martin'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM drivers; +~~~ +~~~ + id | city | name ++--------------------------------------+----------+---------------+ + 146eebc4-c913-4678-8ea3-c5797d2b7f83 | new york | Harry Potter + 43cafd3b-2537-4fd8-a987-8138f88a22a4 | seattle | Evelyn Martin +(2 rows) +~~~ + +### Create a copy of an existing table + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny_copy AS TABLE users_ny; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny_copy; +~~~ +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 + 19999999-9999-4a00-8000-000000000005 | new york | Nicole Mcmahon | 11540 Patton Extensions | 0303726947 +(6 rows) +~~~ + +When a table copy is created this way, the copy is not associated to +any primary key, secondary index, or constraint that was present on the +original table. + +### Specify a primary key + +You can specify the [primary key](primary-key.html) of a new table created from a selection query: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny_pk (id, city, name PRIMARY KEY) AS SELECT id, city, name FROM users WHERE city = 'new york'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny_pk; +~~~ +~~~ + id | city | name ++--------------------------------------+----------+------------------+ + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White + 19999999-9999-4a00-8000-000000000005 | new york | Nicole Mcmahon + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users_ny_pk; +~~~ +~~~ + table_name | create_statement ++----------------+--------------------------------------------------+ + users_ny_extra | CREATE TABLE users_ny_extra ( + | id UUID NULL, + | city VARCHAR NULL, + | name VARCHAR NOT NULL, + | CONSTRAINT "primary" PRIMARY KEY (name ASC), + | FAMILY "primary" (id, city, name) + | ) +(1 row) +~~~ + +### Define column families + +You can define the [column families](column-families.html) of a new table created from a selection query: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny_alt (id PRIMARY KEY FAMILY ids, name, city FAMILY locs, address, credit_card FAMILY payments) AS SELECT id, name, city, address, credit_card FROM users WHERE city = 'new york'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny_alt; +~~~ +~~~ + id | name | city | address | credit_card ++--------------------------------------+------------------+----------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | Robert Murphy | new york | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | James Hamilton | new york | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | Judy White | new york | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | Devin Jordan | new york | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | Catherine Nelson | new york | 1149 Lee Alley | 0792553487 + 19999999-9999-4a00-8000-000000000005 | Nicole Mcmahon | new york | 11540 Patton Extensions | 0303726947 +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users_ny_alt; +~~~ +~~~ + table_name | create_statement ++--------------+------------------------------------------------+ + users_ny_alt | CREATE TABLE users_ny_alt ( + | id UUID NOT NULL, + | name VARCHAR NULL, + | city VARCHAR NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (id ASC), + | FAMILY ids (id, name, address), + | FAMILY locs (city), + | FAMILY payments (credit_card) + | ) +(1 row) +~~~ + +### Specify a primary key for partitioning + +If you are [partitioning](partitioning.html) a table based on a [primary key](primary-key.html), the primary key must be properly defined. To change the primary key after table creation, you can use an [`ALTER TABLE ... ALTER PRIMARY KEY`](alter-primary-key.html) statement. + +Suppose that you want to [geo-partition](demo-low-latency-multi-region-deployment.html) the `drivers` table that you created with the following statement: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE drivers (id, city, name) AS VALUES (gen_random_uuid(), 'new york', 'Harry Potter'), (gen_random_uuid(), 'seattle', 'Evelyn Martin'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE drivers; +~~~ +~~~ + table_name | create_statement ++------------+----------------------------------------------+ + drivers | CREATE TABLE drivers ( + | id UUID NULL, + | city STRING NULL, + | name STRING NULL, + | FAMILY "primary" (id, city, name, rowid) + | ) +(1 row) +~~~ + +In order for this table to be properly geo-partitioned with the other tables in the `movr` dataset, the table must have a composite primary key defined that includes the unique row identifier (`id`, in this case) and the row locality identifier (`city`). Use the following statement to change the primary key to a composite primary key: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE drivers_pk (id, city, name, PRIMARY KEY (id, city)) AS SELECT id, city, name FROM drivers; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE drivers_pk; +~~~ +~~~ + table_name | create_statement ++------------+----------------------------------------------------------+ + drivers_pk | CREATE TABLE drivers_pk ( + | id UUID NOT NULL, + | city STRING NOT NULL, + | name STRING NULL, + | CONSTRAINT "primary" PRIMARY KEY (id ASC, city ASC), + | FAMILY "primary" (id, city, name) + | ) +(1 row) +~~~ + +## See also + +- [Selection Queries](selection-queries.html) +- [Simple `SELECT` Clause](select-clause.html) +- [`CREATE TABLE`](create-table.html) +- [`CREATE VIEW`](create-view.html) +- [`INSERT`](insert.html) +- [`DROP TABLE`](drop-table.html) +- [Other SQL Statements](sql-statements.html) +- [`ALTER PRIMARY KEY`](alter-primary-key.html) +- [`ALTER TABLE`](alter-table.html) diff --git a/v20.2/create-table.md b/v20.2/create-table.md new file mode 100644 index 00000000000..62c11066dc7 --- /dev/null +++ b/v20.2/create-table.md @@ -0,0 +1,638 @@ +--- +title: CREATE TABLE +summary: The CREATE TABLE statement creates a new table in a database. +toc: true +--- + +The `CREATE TABLE` [statement](sql-statements.html) creates a new table in a database. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database. + +## Synopsis + +
+ + +

+ +
+{% include {{ page.version.version }}/sql/diagrams/create_table.html %} +
+ +
+ +
+ {% include {{ page.version.version }}/sql/diagrams/create_table.html %} +
+ +**opt_temp_create_table ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/opt_temp_create_table.html %} +
+ +**column_def ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/column_def.html %} +
+ +**col_qualification ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/col_qualification.html %} +
+ +**index_def ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/index_def.html %} +
+ +**family_def ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/family_def.html %} +
+ +**table_constraint ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/table_constraint.html %} +
+ +**opt_interleave ::=** + +
+ {% include {{ page.version.version }}/sql/diagrams/opt_interleave.html %} +
+ +
+ +{{site.data.alerts.callout_success}}To create a table from the results of a SELECT statement, use CREATE TABLE AS. +{{site.data.alerts.end}} + +## Parameters + +Parameter | Description +----------|------------ +`IF NOT EXISTS` | Create a new table only if a table of the same name does not already exist in the database; if one does exist, do not return an error.

Note that `IF NOT EXISTS` checks the table name only; it does not check if an existing table has the same columns, indexes, constraints, etc., of the new table. +`table_name` | The name of the table to create, which must be unique within its database and follow these [identifier rules](keywords-and-identifiers.html#identifiers). When the parent database is not set as the default, the name must be formatted as `database.name`.

The [`UPSERT`](upsert.html) and [`INSERT ON CONFLICT`](insert.html) statements use a temporary table called `excluded` to handle uniqueness conflicts during execution. It's therefore not recommended to use the name `excluded` for any of your tables. +`column_def` | A comma-separated list of column definitions. Each column requires a [name/identifier](keywords-and-identifiers.html#identifiers) and [data type](data-types.html); optionally, a [column-level constraint](constraints.html) or other column qualification (e.g., [computed columns](computed-columns.html)) can be specified. Column names must be unique within the table but can have the same name as indexes or constraints.

Any `PRIMARY KEY`, `UNIQUE`, and `CHECK` [constraints](constraints.html) defined at the column level are moved to the table-level as part of the table's creation. Use the [`SHOW CREATE`](show-create.html) statement to view them at the table level. +`index_def` | An optional, comma-separated list of [index definitions](indexes.html). For each index, the column(s) to index must be specified; optionally, a name can be specified. Index names must be unique within the table and follow these [identifier rules](keywords-and-identifiers.html#identifiers). See the [Create a Table with Secondary Indexes and Inverted Indexes](#create-a-table-with-secondary-and-inverted-indexes) example below.

To enable [hash-sharded indexes](indexes.html#hash-sharded-indexes), set the `experimental_enable_hash_sharded_indexes` [session variable](set-vars.html) to `on`. For examples, see [Create a table with hash-sharded indexes](#create-a-table-with-a-hash-sharded-primary-index) below.

The [`CREATE INDEX`](create-index.html) statement can be used to create an index separate from table creation. +`family_def` | An optional, comma-separated list of [column family definitions](column-families.html). Column family names must be unique within the table but can have the same name as columns, constraints, or indexes.

A column family is a group of columns that are stored as a single key-value pair in the underlying key-value store. CockroachDB automatically groups columns into families to ensure efficient storage and performance. However, there are cases when you may want to manually assign columns to families. For more details, see [Column Families](column-families.html). +`table_constraint` | An optional, comma-separated list of [table-level constraints](constraints.html). Constraint names must be unique within the table but can have the same name as columns, column families, or indexes. +`opt_interleave` | You can potentially optimize query performance by [interleaving tables](interleave-in-parent.html), which changes how CockroachDB stores your data.
{{site.data.alerts.callout_info}}[Hash-sharded indexes](indexes.html#hash-sharded-indexes) cannot be interleaved.{{site.data.alerts.end}} +`opt_partition_by` | An [enterprise-only](enterprise-licensing.html) option that lets you define table partitions at the row level. You can define table partitions by list or by range. See [Define Table Partitions](partitioning.html) for more information. +`opt_temp_create_table` | Defines the table as a session-scoped temporary table. For more information, see [Temporary Tables](temporary-tables.html).

**Support for temporary tables is [experimental](experimental-features.html#temporary-objects)**. + +## Table-level replication + +By default, tables are created in the default replication zone but can be placed into a specific replication zone. See [Create a Replication Zone for a Table](configure-replication-zones.html#create-a-replication-zone-for-a-table) for more information. + +## Row-level replication + +CockroachDB allows [enterprise users](enterprise-licensing.html) to [define table partitions](partitioning.html), thus providing row-level control of how and where the data is stored. See [Create a Replication Zone for a Table Partition](configure-replication-zones.html#create-a-replication-zone-for-a-partition) for more information. + +{{site.data.alerts.callout_info}}The primary key required for partitioning is different from the conventional primary key. To define the primary key for partitioning, prefix the unique identifier(s) in the primary key with all columns you want to partition and subpartition the table on, in the order in which you want to nest your subpartitions. See Partition using Primary Key for more details.{{site.data.alerts.end}} + +## Examples + +### Create a table + +In this example, we create the `users` table with a single [primary key](primary-key.html) column defined. In CockroachDB, every table requires a [primary key](primary-key.html). If one is not explicitly defined, a column called `rowid` of the type `INT` is added automatically as the primary key, with the `unique_rowid()` function used to ensure that new rows always default to unique `rowid` values. The primary key is automatically indexed. + +{{site.data.alerts.callout_info}} + If no primary key is explicitly defined in a `CREATE TABLE` statement, you can add a primary key to the table with [`ADD CONSTRAINT ... PRIMARY KEY`](add-constraint.html) or [`ALTER PRIMARY KEY`](alter-primary-key.html). If the `ADD` or `ALTER` statement follows the `CREATE TABLE` statement, and is part of the same transaction, no default primary key will be created. If the table has already been created and the transaction committed, the `ADD` or `ALTER` statements replace the default primary key. + {{site.data.alerts.end}} + +{{site.data.alerts.callout_info}}Strictly speaking, a primary key's unique index is not created; it is derived from the key(s) under which the data is stored, so it takes no additional space. However, it appears as a normal unique index when using commands like SHOW INDEX.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID PRIMARY KEY, + city STRING, + name STRING, + address STRING, + credit_card STRING, + dl STRING +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM users; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden ++-------------+-----------+-------------+----------------+-----------------------+-----------+-----------+ + id | UUID | false | NULL | | {primary} | false + city | STRING | true | NULL | | {} | false + name | STRING | true | NULL | | {} | false + address | STRING | true | NULL | | {} | false + credit_card | STRING | true | NULL | | {} | false + dl | STRING | true | NULL | | {} | false +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM users; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit ++------------+------------+------------+--------------+-------------+-----------+---------+----------+ + users | primary | false | 1 | id | ASC | false | false +(1 row) +~~~ + +### Create a table with secondary and inverted indexes + +In this example, we create secondary and inverted indexes during table creation. Secondary indexes allow efficient access to data with keys other than the primary key. [Inverted indexes](inverted-indexes.html) allow efficient access to the schemaless data in a [`JSONB`](jsonb.html) column. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE vehicles ( + id UUID NOT NULL, + city STRING NOT NULL, + type STRING, + owner_id UUID, + creation_time TIMESTAMP, + status STRING, + current_location STRING, + ext JSONB, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + INDEX index_status (status), + INVERTED INDEX ix_vehicle_ext (ext), + FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM vehicles; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit +-------------+----------------+------------+--------------+-------------+-----------+---------+----------- + vehicles | primary | false | 1 | city | ASC | false | false + vehicles | primary | false | 2 | id | ASC | false | false + vehicles | index_status | true | 1 | status | ASC | false | false + vehicles | index_status | true | 2 | city | ASC | false | true + vehicles | index_status | true | 3 | id | ASC | false | true + vehicles | ix_vehicle_ext | true | 1 | ext | ASC | false | false + vehicles | ix_vehicle_ext | true | 2 | city | ASC | false | true + vehicles | ix_vehicle_ext | true | 3 | id | ASC | false | true +(8 rows) +~~~ + +We also have other resources on indexes: + +- Create indexes for existing tables using [`CREATE INDEX`](create-index.html). +- [Learn more about indexes](indexes.html). + +### Create a table with auto-generated unique row IDs + +{% include {{ page.version.version }}/faq/auto-generate-unique-ids.html %} + +### Create a table with a foreign key constraint + +[Foreign key constraints](foreign-key.html) guarantee a column uses only values that already exist in the column it references, which must be from another table. This constraint enforces referential integrity between the two tables. + +There are a [number of rules](foreign-key.html#rules-for-creating-foreign-keys) that govern foreign keys, but the two most important are: + +- Foreign key columns must be [indexed](indexes.html). If no index is defined in the `CREATE TABLE` statement using `INDEX`, `PRIMARY KEY`, or `UNIQUE`, a secondary index is automatically created on the foreign key columns. + +- Referenced columns must contain only unique values. This means the `REFERENCES` clause must use exactly the same columns as a [primary key](primary-key.html) or [unique](unique.html) constraint. + +You can include a [foreign key action](foreign-key.html#foreign-key-actions) to specify what happens when a column referenced by a foreign key constraint is updated or deleted. The default actions are `ON UPDATE NO ACTION` and `ON DELETE NO ACTION`. + +In this example, we use `ON DELETE CASCADE` (i.e., when row referenced by a foreign key constraint is deleted, all dependent rows are also deleted). + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + city STRING, + name STRING, + address STRING, + credit_card STRING, + dl STRING UNIQUE CHECK (LENGTH(dl) < 8) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE vehicles ( + id UUID NOT NULL DEFAULT gen_random_uuid(), + city STRING NOT NULL, + type STRING, + owner_id UUID REFERENCES users(id) ON DELETE CASCADE, + creation_time TIMESTAMP, + status STRING, + current_location STRING, + ext JSONB, + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC), + INVERTED INDEX ix_vehicle_ext (ext), + FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE vehicles; +~~~ + +~~~ + table_name | create_statement ++------------+-----------------------------------------------------------------------------------------------------+ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL DEFAULT gen_random_uuid(), + | city STRING NOT NULL, + | type STRING NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status STRING NULL, + | current_location STRING NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC), + | INVERTED INDEX ix_vehicle_ext (ext), + | CONSTRAINT fk_owner_id_ref_users FOREIGN KEY (owner_id) REFERENCES users(id) ON DELETE CASCADE, + | INDEX vehicles_auto_index_fk_owner_id_ref_users (owner_id ASC), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO users (name, dl) VALUES ('Annika', 'ABC-123'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users; +~~~ + +~~~ + id | city | name | address | credit_card | dl ++--------------------------------------+------+--------+---------+-------------+---------+ + 26da1fce-59e1-4290-a786-9068242dd195 | NULL | Annika | NULL | NULL | ABC-123 +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO vehicles (city, owner_id) VALUES ('seattle', '26da1fce-59e1-4290-a786-9068242dd195'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM vehicles; +~~~ + +~~~ + id | city | type | owner_id | creation_time | status | current_location | ext ++--------------------------------------+---------+------+--------------------------------------+---------------+--------+------------------+------+ + fc6f7a8c-4ba9-42e1-9c37-7be3c906050c | seattle | NULL | 26da1fce-59e1-4290-a786-9068242dd195 | NULL | NULL | NULL | NULL +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM users WHERE id = '26da1fce-59e1-4290-a786-9068242dd195'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM vehicles; +~~~ +~~~ + id | city | type | owner_id | creation_time | status | current_location | ext ++----+------+------+----------+---------------+--------+------------------+-----+ +(0 rows) +~~~ + + +### Create a table with a check constraint + +In this example, we create the `users` table, but with some column [constraints](constraints.html). One column is the [primary key](primary-key.html), and another column is given a [unique constraint](unique.html) and a [check constraint](check.html) that limits the length of the string. Primary key columns and columns with unique constraints are automatically indexed. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users ( + id UUID PRIMARY KEY, + city STRING, + name STRING, + address STRING, + credit_card STRING, + dl STRING UNIQUE CHECK (LENGTH(dl) < 8) +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM users; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden ++-------------+-----------+-------------+----------------+-----------------------+------------------------+-----------+ + id | UUID | false | NULL | | {primary,users_dl_key} | false + city | STRING | true | NULL | | {} | false + name | STRING | true | NULL | | {} | false + address | STRING | true | NULL | | {} | false + credit_card | STRING | true | NULL | | {} | false + dl | STRING | true | NULL | | {users_dl_key} | false +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM users; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit ++------------+--------------+------------+--------------+-------------+-----------+---------+----------+ + users | primary | false | 1 | id | ASC | false | false + users | users_dl_key | false | 1 | dl | ASC | false | false + users | users_dl_key | false | 2 | id | ASC | false | true +(3 rows) +~~~ + +### Create a table that mirrors key-value storage + +{% include {{ page.version.version }}/faq/simulate-key-value-store.html %} + +### Create a table from a `SELECT` statement + +You can use the [`CREATE TABLE AS`](create-table-as.html) statement to create a new table from the results of a `SELECT` statement. For example, suppose you have a number of rows of user data in the `users` table, and you want to create a new table from the subset of users that are located in New York. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users WHERE city = 'new york'; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 +(5 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE users_ny AS SELECT * FROM users WHERE city = 'new york'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM users_ny; +~~~ + +~~~ + id | city | name | address | credit_card ++--------------------------------------+----------+------------------+-----------------------------+-------------+ + 00000000-0000-4000-8000-000000000000 | new york | Robert Murphy | 99176 Anderson Mills | 8885705228 + 051eb851-eb85-4ec0-8000-000000000001 | new york | James Hamilton | 73488 Sydney Ports Suite 57 | 8340905892 + 0a3d70a3-d70a-4d80-8000-000000000002 | new york | Judy White | 18580 Rosario Ville Apt. 61 | 2597958636 + 0f5c28f5-c28f-4c00-8000-000000000003 | new york | Devin Jordan | 81127 Angela Ferry Apt. 8 | 5614075234 + 147ae147-ae14-4b00-8000-000000000004 | new york | Catherine Nelson | 1149 Lee Alley | 0792553487 +(5 rows) +~~~ + +### Create a table with a computed column + +{% include {{ page.version.version }}/computed-columns/simple.md %} + +### Create a table with partitions + +{{site.data.alerts.callout_info}} +The primary key required for partitioning is different from the conventional primary key. To define the primary key for partitioning, prefix the unique identifier(s) in the primary key with all columns you want to partition and subpartition the table on, in the order in which you want to nest your subpartitions. See [Partition using Primary Key](partitioning.html#partition-using-primary-key) for more details. +{{site.data.alerts.end}} + +#### Create a table with partitions by list + +In this example, we create a table and [define partitions by list](partitioning.html#partition-by-list). + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE rides ( + id UUID NOT NULL, + city STRING NOT NULL, + vehicle_city STRING, + rider_id UUID, + vehicle_id UUID, + start_address STRING, + end_address STRING, + start_time TIMESTAMP, + end_time TIMESTAMP, + revenue DECIMAL(10,2), + CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + INDEX rides_auto_index_fk_city_ref_users (city ASC, rider_id ASC), + INDEX rides_auto_index_fk_vehicle_city_ref_vehicles (vehicle_city ASC, vehicle_id ASC), + FAMILY "primary" (id, city, vehicle_city, rider_id, vehicle_id, start_address, end_address, start_time, end_time, revenue), + CONSTRAINT check_vehicle_city_city CHECK (vehicle_city = city)) + PARTITION BY LIST (city) + (PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle')); +~~~ + +#### Create a table with partitions by range + +In this example, we create a table and [define partitions by range](partitioning.html#partition-by-range). + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE rides ( + id UUID NOT NULL, + city STRING NOT NULL, + vehicle_city STRING, + rider_id UUID, + vehicle_id UUID, + start_address STRING, + end_address STRING, + start_time TIMESTAMP, + end_time TIMESTAMP, + ride_length INTERVAL as (start_time - end_time) STORED, + revenue DECIMAL(10,2), + CONSTRAINT "primary" PRIMARY KEY (ride_length ASC, city ASC, id ASC), + INDEX rides_auto_index_fk_city_ref_users (city ASC, rider_id ASC), + INDEX rides_auto_index_fk_vehicle_city_ref_vehicles (vehicle_city ASC, vehicle_id ASC), + FAMILY "primary" (id, city, vehicle_city, rider_id, vehicle_id, start_address, end_address, start_time, end_time, revenue), + CONSTRAINT check_vehicle_city_city CHECK (vehicle_city = city)) + PARTITION BY RANGE (ride_length) + (PARTITION short_rides VALUES FROM ('0 seconds') TO ('30 minutes'), + PARTITION long_rides VALUES FROM ('30 minutes') TO (MAXVALUE)); +~~~ + +### Show the definition of a table + +To show the definition of a table, use the [`SHOW CREATE`](show-create.html) statement. The contents of the `create_statement` column in the response is a string with embedded line breaks that, when echoed, produces formatted output. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE rides; +~~~ + +~~~ + table_name | create_statement ++------------+----------------------------------------------------------------------------------------------------------------------------------------------+ + rides | CREATE TABLE rides ( + | id UUID NOT NULL, + | city STRING NOT NULL, + | vehicle_city STRING NULL, + | rider_id UUID NULL, + | vehicle_id UUID NULL, + | start_address STRING NULL, + | end_address STRING NULL, + | start_time TIMESTAMP NULL, + | end_time TIMESTAMP NULL, + | ride_length INTERVAL NOT NULL AS (start_time - end_time) STORED, + | revenue DECIMAL(10,2) NULL, + | CONSTRAINT "primary" PRIMARY KEY (ride_length ASC, city ASC, id ASC), + | INDEX rides_auto_index_fk_city_ref_users (city ASC, rider_id ASC), + | INDEX rides_auto_index_fk_vehicle_city_ref_vehicles (vehicle_city ASC, vehicle_id ASC), + | FAMILY "primary" (id, city, vehicle_city, rider_id, vehicle_id, start_address, end_address, start_time, end_time, revenue, ride_length), + | CONSTRAINT check_vehicle_city_city CHECK (vehicle_city = city) + | ) PARTITION BY RANGE (ride_length) ( + | PARTITION short_rides VALUES FROM ('00:00:00') TO ('00:30:00'), + | PARTITION long_rides VALUES FROM ('00:30:00') TO (MAXVALUE) + | ) +(1 row) +~~~ + +### Create a table with a hash-sharded primary index + +{% include {{page.version.version}}/performance/use-hash-sharded-indexes.md %} + +{% include copy-clipboard.html %} +~~~ sql +> SET experimental_enable_hash_sharded_indexes=on; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE events ( + ts DECIMAL PRIMARY KEY USING HASH WITH BUCKET_COUNT=8, + product_id INT8 + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM events; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden +---------------------------+-----------+-------------+----------------+-----------------------------------+-----------+------------ + crdb_internal_ts_shard_8 | INT4 | false | NULL | mod(fnv32(CAST(ts AS STRING)), 8) | {primary} | true + ts | DECIMAL | false | NULL | | {primary} | false + product_id | INT8 | true | NULL | | {} | false +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM events; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit +-------------+------------+------------+--------------+--------------------------+-----------+---------+----------- + events | primary | false | 1 | crdb_internal_ts_shard_8 | ASC | false | false + events | primary | false | 2 | ts | ASC | false | false +(2 rows) +~~~ + +### Create a table with a hash-sharded secondary index + +{% include copy-clipboard.html %} +~~~ sql +> SET experimental_enable_hash_sharded_indexes=on; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE events ( + product_id INT8, + owner UUID, + serial_number VARCHAR, + event_id UUID, + ts TIMESTAMP, + data JSONB, + PRIMARY KEY (product_id, owner, serial_number, ts, event_id), + INDEX (ts) USING HASH WITH BUCKET_COUNT=8 +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM events; +~~~ + +~~~ + table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit +-------------+----------------------------------------+------------+--------------+--------------------------+-----------+---------+----------- + events | primary | false | 1 | product_id | ASC | false | false + events | primary | false | 2 | owner | ASC | false | false + events | primary | false | 3 | serial_number | ASC | false | false + events | primary | false | 4 | ts | ASC | false | false + events | primary | false | 5 | event_id | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 1 | crdb_internal_ts_shard_8 | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 2 | ts | ASC | false | false + events | events_crdb_internal_ts_shard_8_ts_idx | true | 3 | product_id | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 4 | owner | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 5 | serial_number | ASC | false | true + events | events_crdb_internal_ts_shard_8_ts_idx | true | 6 | event_id | ASC | false | true +(11 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM events; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden +---------------------------+-----------+-------------+----------------+-----------------------------------+--------------------------------------------------+------------ + product_id | INT8 | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + owner | UUID | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + serial_number | VARCHAR | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + event_id | UUID | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + ts | TIMESTAMP | false | NULL | | {primary,events_crdb_internal_ts_shard_8_ts_idx} | false + data | JSONB | true | NULL | | {} | false + crdb_internal_ts_shard_8 | INT4 | false | NULL | mod(fnv32(CAST(ts AS STRING)), 8) | {events_crdb_internal_ts_shard_8_ts_idx} | true +(7 rows) +~~~ + +## See also + +- [`INSERT`](insert.html) +- [`ALTER TABLE`](alter-table.html) +- [`DELETE`](delete.html) +- [`DROP TABLE`](drop-table.html) +- [`RENAME TABLE`](rename-table.html) +- [`SHOW TABLES`](show-tables.html) +- [`SHOW COLUMNS`](show-columns.html) +- [Column Families](column-families.html) +- [Table-Level Replication Zones](configure-replication-zones.html#create-a-replication-zone-for-a-table) +- [Define Table Partitions](partitioning.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/create-user.md b/v20.2/create-user.md new file mode 100644 index 00000000000..0ae3d6d5f94 --- /dev/null +++ b/v20.2/create-user.md @@ -0,0 +1,237 @@ +--- +title: CREATE USER +summary: The CREATE USER statement creates SQL users, which let you control privileges on your databases and tables. +toc: true +--- + +The `CREATE USER` [statement](sql-statements.html) creates SQL users, which let you control [privileges](authorization.html#assign-privileges) on your databases and tables. + +{{site.data.alerts.callout_info}} + Since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `CREATE USER` is now an alias for [`CREATE ROLE`](create-role.html). +{{site.data.alerts.end}} + +## Considerations + +- Usernames: + - Are case-insensitive + - Must start with a letter, number, or underscore + - Must contain only letters, numbers, periods, or underscores + - Must be between 1 and 63 characters. +- After creating users, you must [grant them privileges to databases and tables](grant.html). +- All users belong to the `public` role, to which you can [grant](grant.html) and [revoke](revoke.html) privileges. +- On secure clusters, you must [create client certificates for users](cockroach-cert.html#create-the-certificate-and-key-pair-for-a-client) and users must [authenticate their access to the cluster](#user-authentication). + +## Required privileges + + To create other users, the user must have the [`CREATEROLE`](#allow-the-user-to-create-other-users) parameter set. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/create_user.html %}
+ +## Parameters + + + + Parameter | Description +-----------|------------- +`user_name` | The name of the user you want to create.

Usernames are case-insensitive; must start with a letter, number, or underscore; must contain only letters, numbers, or underscores; and must be between 1 and 63 characters. +`password` | Let the user [authenticate their access to a secure cluster](authentication.html#client-authentication) using this password. Passwords should be entered as a [string literal](sql-constants.html#string-literals). For compatibility with PostgreSQL, a password can also be entered as an [identifier](#create-a-user-with-a-password-using-an-identifier).

To prevent a user from using [password authentication](authentication.html#client-authentication) and to mandate [certificate-based client authentication](authentication.html#client-authentication), [set the password as `NULL`](#prevent-a-user-from-using-password-authentication). +`VALID UNTIL` | The date and time (in the [`timestamp`](timestamp.html) format) after which the password is not valid. +`LOGIN`/`NOLOGIN` | The `LOGIN` parameter allows a user to login with one of the [user authentication methods](#user-authentication). [Setting the parameter to `NOLOGIN`](#set-login-privileges-for-a-user) prevents the user from logging in using any authentication method.

By default, the parameter is set to `LOGIN` for the `CREATE USER` statement. +`CREATEROLE`/`NOCREATEROLE` | Allow or disallow the new user to create, alter, and drop other users.

By default, the parameter is set to `NOCREATEROLE` for all non-admin and non-root users. + +## User authentication + +Secure clusters require users to authenticate their access to databases and tables. CockroachDB offers three methods for this: + +- [Client certificate and key authentication](#secure-clusters-with-client-certificates), which is available to all users. To ensure the highest level of security, we recommend only using client certificate and key authentication. + +- [Password authentication](#secure-clusters-with-passwords), which is available to users and roles who you've created passwords for. To create a user with a password, use the `WITH PASSWORD` clause of `CREATE USER`. To add a password to an existing user, use the [`ALTER USER`](alter-user.html) statement. + + Users can use passwords to authenticate without supplying client certificates and keys; however, we recommend using certificate-based authentication whenever possible. + + Password creation is supported only in secure clusters. + +- [GSSAPI authentication](gssapi_authentication.html), which is available to [Enterprise users](enterprise-licensing.html). + +## Examples + +### Create a user + +Usernames are case-insensitive; must start with a letter, number, or underscore; must contain only letters, numbers, periods, or underscores; and must be between 1 and 63 characters. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl; +~~~ + +After creating users, you must: + +- [Grant them privileges to databases](grant.html). +- For secure clusters, you must also [create their client certificates](cockroach-cert.html#create-the-certificate-and-key-pair-for-a-client). + +### Allow the user to create other users + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl with CREATEROLE; +~~~ + +### Create a user with a password + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl WITH PASSWORD 'ilov3beefjerky'; +~~~ + +~~~ +CREATE USER 1 +~~~ + +### Create a user with a password using an identifier + +The following statement changes the password to `ilov3beefjerky`, as above: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl WITH PASSWORD ilov3beefjerky; +~~~ + +This is equivalent to the example in the previous section because the password contains only lowercase characters. + +In contrast, the following statement changes the password to `thereisnotomorrow`, even though the password in the syntax contains capitals, because identifiers are normalized automatically: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl WITH PASSWORD ThereIsNoTomorrow; +~~~ + +To preserve case in a password specified using identifier syntax, use double quotes: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl WITH PASSWORD "ThereIsNoTomorrow"; +~~~ + +### Prevent a user from using password authentication + +The following statement prevents the user from using password authentication and mandates certificate-based client authentication: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl WITH PASSWORD NULL; +~~~ + +### Set password validity + +The following statement sets the date and time after which the password is not valid: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl VALID UNTIL '2021-01-01'; +~~~ + +### Manage users + +After creating a user, you can use the [`ALTER USER`](alter-user.html) statement to add or change the user's password, update role options, and the [`DROP USER`](drop-user.html) statement to the remove users. + +### Authenticate as a specific user + +
+ + +
+

+ +
+ +#### Secure clusters with client certificates + +All users can authenticate their access to a secure cluster using [a client certificate](cockroach-cert.html#create-the-certificate-and-key-pair-for-a-client) issued to their username. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --user=carl +~~~ + +#### Secure clusters with passwords + +[Users with passwords](#create-a-user) can authenticate their access by entering their password at the command prompt instead of using their client certificate and key. + +If we cannot find client certificate and key files matching the user, we fall back on password authentication. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --user=carl +~~~ + +
+ +
+ +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --user=carl +~~~ + +
+ +### Set login privileges for a user + +The following statement prevents the user from logging in using any [user authentication method](#user-authentication): + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER carl NOLOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW USERS; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | NOLOGIN | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +To allow the user to log in using one of the user authentication methods, use the `ALTER USER` statement: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER USER carl LOGIN; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW USERS; +~~~ + +~~~ + username | options | member_of +-----------+------------+------------ + admin | CREATEROLE | {} + carl | | {} + root | CREATEROLE | {admin} +(3 rows) +~~~ + +## See also + +- [Authorization](authorization.html) +- [`ALTER USER`](alter-user.html) +- [`DROP USER`](drop-user.html) +- [`SHOW USERS`](show-users.html) +- [`GRANT`](grant.html) +- [`SHOW GRANTS`](show-grants.html) +- [Create Security Certificates](cockroach-cert.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/create-view.md b/v20.2/create-view.md new file mode 100644 index 00000000000..d36c25246a7 --- /dev/null +++ b/v20.2/create-view.md @@ -0,0 +1,110 @@ +--- +title: CREATE VIEW +summary: The CREATE VIEW statement creates a . +toc: true +--- + +The `CREATE VIEW` statement creates a new [view](views.html), which is a stored query represented as a virtual table. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the parent database and the `SELECT` privilege on any table(s) referenced by the view. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/create_view.html %}
+ +## Parameters + +Parameter | Description +----------|------------ +`IF NOT EXISTS` | Create a new view only if a view of the same name does not already exist. If one does exist, do not return an error.

Note that `IF NOT EXISTS` checks the view name only. It does not check if an existing view has the same columns as the new view. +`view_name` | The name of the view to create, which must be unique within its database and follow these [identifier rules](keywords-and-identifiers.html#identifiers). When the parent database is not set as the default, the name must be formatted as `database.name`. +`name_list` | An optional, comma-separated list of column names for the view. If specified, these names will be used in the response instead of the columns specified in `AS select_stmt`. +`AS select_stmt` | The [selection query](selection-queries.html) to execute when the view is requested.

Note that it is not currently possible to use `*` to select all columns from a referenced table or view; instead, you must specify specific columns. +`opt_temp` | Defines the view as a session-scoped temporary view. For more information, see [Temporary Views](views.html#temporary-views).

**Support for temporary views is [experimental](experimental-features.html#temporary-objects)**. + +## Example + +{{site.data.alerts.callout_success}} +This example highlights one key benefit to using views: simplifying complex queries. For additional benefits and examples, see [Views](views.html). +{{site.data.alerts.end}} + +Let's say you're using our [sample `startrek` database](cockroach-gen.html#generate-example-data), which contains two tables, `episodes` and `quotes`. There's a foreign key constraint between the `episodes.id` column and the `quotes.episode` column. To count the number of famous quotes per season, you could run the following join: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT startrek.episodes.season, count(*) + FROM startrek.quotes + JOIN startrek.episodes + ON startrek.quotes.episode = startrek.episodes.id + GROUP BY startrek.episodes.season; +~~~ + +~~~ + season | count +---------+-------- + 1 | 78 + 2 | 76 + 3 | 46 +(3 rows) +~~~ + +Alternatively, to make it much easier to run this complex query, you could create a view: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE VIEW startrek.quotes_per_season (season, quotes) + AS SELECT startrek.episodes.season, count(*) + FROM startrek.quotes + JOIN startrek.episodes + ON startrek.quotes.episode = startrek.episodes.id + GROUP BY startrek.episodes.season; +~~~ + +~~~ +CREATE VIEW +~~~ + +The view is then represented as a virtual table alongside other tables in the database: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM startrek; +~~~ + +~~~ + table_name +--------------------- + episodes + quotes + quotes_per_season +(3 rows) +~~~ + +Executing the query is as easy as `SELECT`ing from the view, as you would from a standard table: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM startrek.quotes_per_season; +~~~ + +~~~ + season | quotes +---------+--------- + 1 | 78 + 2 | 76 + 3 | 46 +(3 rows) +~~~ + +## See also + +- [Selection Queries](selection-queries.html) +- [Views](views.html) +- [`SHOW CREATE`](show-create.html) +- [`ALTER VIEW`](alter-view.html) +- [`DROP VIEW`](drop-view.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/data-types.md b/v20.2/data-types.md new file mode 100644 index 00000000000..129304085cc --- /dev/null +++ b/v20.2/data-types.md @@ -0,0 +1,52 @@ +--- +title: Data Types +summary: Learn about the data types supported by CockroachDB. +toc: true +--- + +## Supported types + +CockroachDB supports the following data types. Click a type for more details. + +Type | Description | Example | [Vectorized Execution](vectorized-execution.html) +-----|-------------|---------|---------- +[`ARRAY`](array.html) | A 1-dimensional, 1-indexed, homogeneous array of any non-array data type. | `{"sky","road","car"}` | Not supported +[`BIT`](bit.html) | A string of binary digits (bits). | `B'10010101'` | Not supported +[`BOOL`](bool.html) | A Boolean value. | `true` | Supported +[`BYTES`](bytes.html) | A string of binary characters. | `b'\141\061\142\062\143\063'` | Supported +[`COLLATE`](collate.html) | The `COLLATE` feature lets you sort [`STRING`](string.html) values according to language- and country-specific rules, known as collations. | `'a1b2c3' COLLATE en` | Not supported +[`DATE`](date.html) | A date. | `DATE '2016-01-25'` | Supported +[`DECIMAL`](decimal.html) | An exact, fixed-point number. | `1.2345` | Supported +[`FLOAT`](float.html) | A 64-bit, inexact, floating-point number. | `1.2345` | Supported +[`INET`](inet.html) | An IPv4 or IPv6 address. | `192.168.0.1` | Not supported +[`INT`](int.html) | A signed integer, up to 64 bits. | `12345` | Supported +[`INTERVAL`](interval.html) | A span of time. | `INTERVAL '2h30m30s'` | Supported +[`JSONB`](jsonb.html) | JSON (JavaScript Object Notation) data. | `'{"first_name": "Lola", "last_name": "Dog", "location": "NYC", "online" : true, "friends" : 547}'` | Not supported +[`SERIAL`](serial.html) | A pseudo-type that combines an [integer type](int.html) with a [`DEFAULT` expression](default-value.html). | `148591304110702593` | Not supported +[`STRING`](string.html) | A string of Unicode characters. | `'a1b2c3'` | Supported +[`TIME`
`TIMETZ`](time.html) | `TIME` stores a time of day in UTC.
`TIMETZ` converts `TIME` values with a specified time zone offset from UTC. | `TIME '01:23:45.123456'`
`TIMETZ '01:23:45.123456-5:00'` | Not supported +[`TIMESTAMP`
`TIMESTAMPTZ`](timestamp.html) | `TIMESTAMP` stores a date and time pairing in UTC.
`TIMESTAMPTZ` converts `TIMESTAMP` values with a specified time zone offset from UTC. | `TIMESTAMP '2016-01-25 10:10:10'`
`TIMESTAMPTZ '2016-01-25 10:10:10-05:00'` | Supported +[`UUID`](uuid.html) | A 128-bit hexadecimal value. | `7f9c24e8-3b12-4fef-91e0-56a2d5a246ec` | Supported + +## Data type conversions and casts + +CockroachDB supports explicit type conversions using the following methods: + +- ` 'string literal'`, to convert from the literal representation of a value to a value of that type. For example: + `DATE '2008-12-21'`, `INT '123'`, or `BOOL 'true'`. + +- `::`, or its equivalent longer form `CAST( AS )`, which converts an arbitrary expression of one built-in type to another (this is also known as type coercion or "casting"). For example: + `NOW()::DECIMAL`, `VARIANCE(a+2)::INT`. + + {{site.data.alerts.callout_success}} + To create constant values, consider using a + type annotation + instead of a cast, as it provides more predictable results. + {{site.data.alerts.end}} + +- Other [built-in conversion functions](functions-and-operators.html) when the type is not a SQL type, for example `from_ip()`, `to_ip()` to convert IP addresses between `STRING` and `BYTES` values. + + CockroachDB also supports implicit casting from string literals to `INT` and `DECIMAL` [`ARRAY`](array.html)s, where appropriate. For an example, see [Implicit casting to `INT` and `DECIMAL` `ARRAY`s](array.html#implicit-casting-to-int-and-decimal-arrays). + +You can find each data type's supported conversion and casting on its +respective page in its section **Supported casting & conversion**. diff --git a/v20.2/date.md b/v20.2/date.md new file mode 100644 index 00000000000..cf99908e15f --- /dev/null +++ b/v20.2/date.md @@ -0,0 +1,95 @@ +--- +title: DATE +summary: The DATE data type stores a year, month, and day. +toc: true +--- + +The `DATE` [data type](data-types.html) stores a year, month, and day. + +## Syntax + +A constant value of type `DATE` can be expressed using an +[interpreted literal](sql-constants.html#interpreted-literals), or a +string literal +[annotated with](scalar-expressions.html#explicitly-typed-expressions) +type `DATE` or +[coerced to](scalar-expressions.html#explicit-type-coercions) type +`DATE`. + +The string format for dates is `YYYY-MM-DD`. For example: `DATE '2016-12-23'`. + +CockroachDB also supports using uninterpreted +[string literals](sql-constants.html#string-literals) in contexts +where a `DATE` value is otherwise expected. + +{{site.data.alerts.callout_info}} +`DATE` values in CockroachDB are fully [PostgreSQL-compatible](https://www.postgresql.org/docs/current/datatype-datetime.html), including support for special values (e.g. `+/- infinity`). Existing dates outside of the PostgreSQL date range (`4714-11-24 BC` to `5874897-12-31`) are converted to `+/- infinity` dates. +{{site.data.alerts.end}} + +## Size + +A `DATE` column supports values up to 16 bytes in width, but the total storage size is likely to be larger due to CockroachDB metadata. + +## Examples + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE dates (a DATE PRIMARY KEY, b INT); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM dates; +~~~ + +~~~ ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +| a | DATE | false | NULL | | {"primary"} | +| b | INT | true | NULL | | {} | ++-------------+-----------+-------------+----------------+-----------------------+-------------+ +(2 rows) +~~~ + +Explicitly typed `DATE` literal: +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO dates VALUES (DATE '2016-03-26', 12345); +~~~ + +String literal implicitly typed as `DATE`: +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO dates VALUES ('2016-03-27', 12345); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM dates; +~~~ + +~~~ ++---------------------------+-------+ +| a | b | ++---------------------------+-------+ +| 2016-03-26 00:00:00+00:00 | 12345 | +| 2016-03-27 00:00:00+00:00 | 12345 | ++---------------------------+-------+ +~~~ + +## Supported casting and conversion + +`DATE` values can be [cast](data-types.html#data-type-conversions-and-casts) to any of the following data types: + +Type | Details +-----|-------- +`DECIMAL` | Converts to number of days since the Unix epoch (Jan. 1, 1970). This is a CockroachDB experimental feature which may be changed without notice. +`FLOAT` | Converts to number of days since the Unix epoch (Jan. 1, 1970). This is a CockroachDB experimental feature which may be changed without notice. +`TIMESTAMP` | Sets the time to 00:00 (midnight) in the resulting timestamp. +`INT` | Converts to number of days since the Unix epoch (Jan. 1, 1970). This is a CockroachDB experimental feature which may be changed without notice. +`STRING` | –– + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/dbeaver.md b/v20.2/dbeaver.md new file mode 100644 index 00000000000..f1109cde0b5 --- /dev/null +++ b/v20.2/dbeaver.md @@ -0,0 +1,90 @@ +--- +title: DBeaver +summary: Learn how to use DBeaver with CockroachDB. +toc: true +--- + +The [DBeaver database tool][dbeaver] is a tool that completely integrates with CockroachDB to provide a GUI for managing your database. + +According to the [DBeaver website][dbeaver]: + +> DBeaver is a cross-platform Database GUI tool for developers, SQL programmers, database administrators, and analysts. + +In this tutorial, you'll work through the process of using DBeaver with a secure CockroachDB cluster. + +{{site.data.alerts.callout_success}} +For more information about using DBeaver, see the [DBeaver documentation](https://dbeaver.io/docs/). + +If you run into problems, please file an issue on the [DBeaver issue tracker](https://github.com/dbeaver/dbeaver/issues). +{{site.data.alerts.end}} + +## Before You Begin + +To work through this tutorial, take the following steps: + +- [Install CockroachDB](install-cockroachdb.html) and [start a secure cluster](secure-a-cluster.html). +- Download a copy of [DBeaver](https://dbeaver.io/download/) version 5.2.3 or greater. + +## Step 1. Start DBeaver and connect to CockroachDB + +Start DBeaver, and select **Database > New Connection** from the menu. In the dialog that appears, select **CockroachDB** from the list. + +DBeaver - Select CockroachDB + +## Step 2. Update the connection settings + +On the **Create new connection** dialog that appears, click **Network settings**. + +DBeaver - CockroachDB connection settings + +From the network settings, click the **SSL** tab. It will look like the screenshot below. + +DBeaver - SSL tab + +Check the **Use SSL** checkbox as shown, and fill in the text areas as follows: + +- **Root certificate**: Use the `ca.crt` file you generated for your secure cluster. +- **SSL certificate**: Use a client certificate generated from your cluster's root certificate. For the root user, this will be named `client.root.crt`. For additional security, you may want to create a new database user and client certificate just for use with DBeaver. +- **SSL certificate key**: Because DBeaver is a Java application, you will need to transform your key file to the `*.pk8` format using an [OpenSSL command](https://wiki.openssl.org/index.php/Command_Line_Utilities#pkcs8_.2F_pkcs5) like the one shown below. Once you have created the file, enter its location here. In this example, the filename is `client.root.pk8`. + {% include copy-clipboard.html %} + ~~~ console + $ openssl pkcs8 -topk8 -inform PEM -outform DER -in client.root.key -out client.root.pk8 -nocrypt + ~~~ + +Select **require** from the **SSL mode** dropdown. There is no need to set the **SSL Factory**, you can let DBeaver use the default. + +## Step 3. Test the connection settings + +Click **Test Connection ...**. If everything worked, you will see a **Success** dialog like the one shown below. + +DBeaver - connection success dialog + +## Step 4. Start using DBeaver + +Click **Finish** to get started using DBeaver with CockroachDB. + +DBeaver - CockroachDB with the movr database + +For more information about using DBeaver, see the [DBeaver documentation](https://dbeaver.io/docs/). + +## Report Issues with DBeaver & CockroachDB + +If you run into problems, please file an issue on the [DBeaver issue tracker](https://github.com/dbeaver/dbeaver/issues), including the following details about the environment where you encountered the issue: + +- CockroachDB version ([`cockroach version`](cockroach-version.html)) +- DBeaver version +- Operating system +- Steps to reproduce the behavior +- If possible, a trace of the SQL statements sent to CockroachDB while the error is being reproduced using [SQL query logging](query-behavior-troubleshooting.html#sql-logging). + +## See Also + ++ [DBeaver documentation](https://dbeaver.io/docs/) ++ [DBeaver issue tracker](https://github.com/dbeaver/dbeaver/issues) ++ [Client connection parameters](connection-parameters.html) ++ [Third-Party Database Tools](third-party-database-tools.html) ++ [Learn CockroachDB SQL](learn-cockroachdb-sql.html) + + + +[dbeaver]: https://dbeaver.io diff --git a/v20.2/debug-and-error-logs.md b/v20.2/debug-and-error-logs.md new file mode 100644 index 00000000000..2eaa35543c5 --- /dev/null +++ b/v20.2/debug-and-error-logs.md @@ -0,0 +1,119 @@ +--- +title: Understand Debug & Error Logs +summary: Learn how to find and read CockroachDB error logs +toc: true +--- + +If you need to [troubleshoot](troubleshooting-overview.html) issues with your cluster, you can check a node's logs, which include details about certain node-level and range-level events, such as errors. For example, if CockroachDB crashes, it normally logs a stack trace to what caused the problem. + +{{site.data.alerts.callout_success}} +CockroachDB also generates secondary logs for queries being executed against your system. See [SQL logging](query-behavior-troubleshooting.html#sql-logging) for details on obtaining these logs. +{{site.data.alerts.end}} + + +## Details + +When a node processes a [`cockroach` command](cockroach-commands.html), it produces a stream of messages about the command's activities. Each message's body describes the activity, and its envelope contains metadata such as the message's severity level. + +As a command generates messages, CockroachDB uses the [command](#commands)'s [logging flags](#flags) and the message's [severity level](#severity-levels) to determine the appropriate [location](#output-locations) for it. + +Each node's logs detail only the internal activity of that node without visibility into the behavior of other nodes in the cluster. When troubleshooting, this means that you must identify the node where the problem occurred or [collect the logs from all active nodes in your cluster](cockroach-debug-zip.html). + +### Commands + +All [`cockroach` commands](cockroach-commands.html) support logging. However, it's important to note: + +- `cockroach start` generates most messages related to the operation of your cluster. +- Other commands do generate messages, but they're typically only interesting in troubleshooting scenarios. + +### Severity levels + +CockroachDB identifies each message with a severity level, letting operators know if they need to intercede: + +1. `INFO` *(lowest severity; no action necessary)* +2. `WARNING` +3. `ERROR` +4. `FATAL` *(highest severity; requires operator attention)* + +**Default behavior by severity level** + +Command | `INFO` messages | `WARNING` and above messages +--------|--------|-------------------- +[`cockroach start`](cockroach-start.html) | Write to file | Write to file +[All other commands](cockroach-commands.html) | Discard | Print to `stderr` + +### Output locations + +Based on the command's flags and the message's [severity level](#severity-levels), CockroachDB does one of the following: + +- [Writes the message to a file](#write-to-file) +- [Prints it to `stderr`](#print-to-stderr) +- [Discards the message entirely](#discard-message) + +#### Write to file + +CockroachDB can write messages to log files. The files are named using the following format: + +~~~ +cockroach.[host].[user].[start timestamp in UTC].[process ID].log +~~~ + +For example: + +~~~ +cockroach.richards-mbp.rloveland.2018-03-15T15_24_10Z.024338.log +~~~ + +To make it easier to watch a log without knowing the full filename, a [symlink](https://en.wikipedia.org/wiki/Symbolic_link) with the short filename `cockroach.log` is also created. This symlink points to the most recent log. + +{{site.data.alerts.callout_info}} +All log file timestamps are in UTC because CockroachDB is designed to be deployed in a distributed cluster. Nodes may be located in different time zones, and using UTC makes it easy to correlate log messages from those nodes no matter where they are located. +{{site.data.alerts.end}} + +Property | `cockroach start` | All other commands +---------|-------------------|------------------- +Enabled by | Default1 | Explicit `--log-dir` flag +Default File Destination | `[first `[`store`](cockroach-start.html#store)` dir]/logs` | *N/A* +Change File Destination | `--log-dir=[destination]` | `--log-dir=[destination]` +Default Severity Level Threshold | `INFO` | *N/A* +Change Severity Threshold | `--log-file-verbosity=[severity level]` | `--log-file-verbosity=[severity level]` +Disabled by | `--log-dir=""`1 | Default + +{{site.data.alerts.callout_info}} +1 If the `cockroach` process does not have access to on-disk storage, `cockroach start` does not write messages to log files; instead it prints all messages to `stderr`. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_success}} +{% include {{ page.version.version }}/admin-ui/admin-ui-log-files.md %} +{{site.data.alerts.end}} + +#### Print to `stderr` + +CockroachDB can print messages to `stderr`, which normally prints them to the machine's terminal but does not store them. + +Property | `cockroach start` | All other commands +---------|-------------------|------------------- +Enabled by | Explicit `--logtostderr` flag2 | Default +Default Severity Level Threshold | *N/A* | `WARNING` +Change Severity Threshold | `--logtostderr=[severity level]` | `--logtostderr=[severity level]` +Disabled by | Default2 | `--logtostderr=NONE` + +{{site.data.alerts.callout_info}}2 cockroach start does not print any messages to stderr unless the cockroach process does not have access to on-disk storage, in which case it defaults to --logtostderr=INFO and prints all messages to stderr.{{site.data.alerts.end}} + +#### Discard message + +Messages with severity levels below the `--logtostderr` and `--log-file-verbosity` flag's values are neither written to files nor printed to `stderr`, so they are discarded. + +By default, commands besides `cockroach start` discard messages with the `INFO` [severity level](#severity-levels). + +## Flags + +These logging flags are used with [`cockroach` commands](cockroach-commands.html). + +{% include {{ page.version.version }}/misc/logging-flags.md %} + +## See also + +- [SQL logging](query-behavior-troubleshooting.html#sql-logging) +- [Troubleshooting Overview](troubleshooting-overview.html) +- [Support Resources](support-resources.html) diff --git a/v20.2/decimal.md b/v20.2/decimal.md new file mode 100644 index 00000000000..ee22f97c576 --- /dev/null +++ b/v20.2/decimal.md @@ -0,0 +1,110 @@ +--- +title: DECIMAL +summary: The DECIMAL data type stores exact, fixed-point numbers. +toc: true +--- + +The `DECIMAL` [data type](data-types.html) stores exact, fixed-point numbers. This type is used when it is important to preserve exact precision, for example, with monetary data. + +## Aliases + +In CockroachDB, the following are aliases for `DECIMAL`: + +- `DEC` +- `NUMERIC` + +## Precision and scale + +To limit a decimal column, use `DECIMAL(precision, scale)`, where `precision` is the **maximum** count of digits both to the left and right of the decimal point and `scale` is the **exact** count of digits to the right of the decimal point. The `precision` must not be smaller than the `scale`. Also note that using `DECIMAL(precision)` is equivalent to `DECIMAL(precision, 0)`. + +When inserting a decimal value: + +- If digits to the right of the decimal point exceed the column's `scale`, CockroachDB rounds to the scale. +- If digits to the right of the decimal point are fewer than the column's `scale`, CockroachDB pads to the scale with `0`s. +- If digits to the left and right of the decimal point exceed the column's `precision`, CockroachDB gives an error. +- If the column's `precision` and `scale` are identical, the inserted value must round to less than 1. + +## Syntax + +A constant value of type `DECIMAL` can be entered as a [numeric literal](sql-constants.html#numeric-literals). +For example: `1.414` or `-1234`. + +The special IEEE754 values for positive infinity, negative infinity +and [NaN (Not-a-Number)](https://en.wikipedia.org/wiki/NaN) cannot be +entered using numeric literals directly and must be converted using an +[interpreted literal](sql-constants.html#interpreted-literals) or an +[explicit conversion](scalar-expressions.html#explicit-type-coercions) +from a string literal instead. + +The following values are recognized: + + Syntax | Value +----------------------------------------|------------------------------------------------ + `inf`, `infinity`, `+inf`, `+infinity` | +∞ + `-inf`, `-infinity` | -∞ + `nan` | [NaN (Not-a-Number)](https://en.wikipedia.org/wiki/NaN) + +For example: + +- `DECIMAL '+Inf'` +- `'-Inf'::DECIMAL` +- `CAST('NaN' AS DECIMAL)` + +## Size + +The size of a `DECIMAL` value is variable, starting at 9 bytes. It's recommended to keep values under 64 kilobytes to ensure performance. Above that threshold, [write amplification](https://en.wikipedia.org/wiki/Write_amplification) and other considerations may cause significant performance degradation. + +## Examples + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE decimals (a DECIMAL PRIMARY KEY, b DECIMAL(10,5), c NUMERIC); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM decimals; +~~~ + +~~~ + column_name | data_type | is_nullable | column_default | generation_expression | indices | is_hidden +--------------+---------------+-------------+----------------+-----------------------+-----------+------------ + a | DECIMAL | false | NULL | | {primary} | false + b | DECIMAL(10,5) | true | NULL | | {} | false + c | DECIMAL | true | NULL | | {} | false +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO decimals VALUES (1.01234567890123456789, 1.01234567890123456789, 1.01234567890123456789); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM decimals; +~~~ + +~~~ + a | b | c +-------------------------+---------+------------------------- + 1.01234567890123456789 | 1.01235 | 1.01234567890123456789 +(1 row) +~~~ + +The value in column `a` matches what was inserted exactly. The value in column `b` has been rounded to the column's scale. The value in column `c` is handled like the value in column `a` because `NUMERIC` is an alias for `DECIMAL`. + +## Supported casting and conversion + +`DECIMAL` values can be [cast](data-types.html#data-type-conversions-and-casts) to any of the following data types: + +Type | Details +-----|-------- +`INT` | Truncates decimal precision +`FLOAT` | Loses precision and may round up to +/- infinity if the value is too large in magnitude, or to +/-0 if the value is too small in magnitude +`BOOL` | **0** converts to `false`; all other values convert to `true` +`STRING` | –– + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/default-value.md b/v20.2/default-value.md new file mode 100644 index 00000000000..2535ab4265e --- /dev/null +++ b/v20.2/default-value.md @@ -0,0 +1,81 @@ +--- +title: Default Value Constraint +summary: The Default Value constraint specifies a value to populate a column with if none is provided. +toc: true +--- + +The `DEFAULT` value [constraint](constraints.html) specifies a value to write into the constrained column if one is not defined in an `INSERT` statement. The value may be either a hard-coded literal or an expression that is evaluated at the time the row is created. + + +## Details + +- The [data type](data-types.html) of the Default Value must be the same as the data type of the column. +- The `DEFAULT` value constraint only applies if the column does not have a value specified in the [`INSERT`](insert.html) statement. You can still insert a *NULL* into an optional (nullable) column by explicitly inserting *NULL*. For example, `INSERT INTO foo VALUES (1, NULL);`. + +## Syntax + +You can only apply the `DEFAULT` value constraint to individual columns. + +{{site.data.alerts.callout_info}} +You can also add the `DEFAULT` value constraint to an existing table through [`ALTER COLUMN`](alter-column.html#set-or-change-a-default-value). +{{site.data.alerts.end}} + +
{% include {{ page.version.version }}/sql/diagrams/default_value_column_level.html %}
+ + Parameter | Description +-----------|------------- + `table_name` | The name of the table you're creating. + `column_name` | The name of the constrained column. + `column_type` | The constrained column's [data type](data-types.html). + `default_value` | The value you want to insert by default, which must evaluate to the same [data type](data-types.html) as the `column_type`. + `column_constraints` | Any other column-level [constraints](constraints.html) you want to apply to this column. + `column_def` | Definitions for any other columns in the table. + `table_constraints` | Any table-level [constraints](constraints.html) you want to apply. + +## Example + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE inventories ( + product_id INT, + warehouse_id INT, + quantity_on_hand INT DEFAULT 100, + PRIMARY KEY (product_id, warehouse_id) + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO inventories (product_id, warehouse_id) VALUES (1,20); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO inventories (product_id, warehouse_id, quantity_on_hand) VALUES (2,30, NULL); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM inventories; +~~~ +~~~ ++------------+--------------+------------------+ +| product_id | warehouse_id | quantity_on_hand | ++------------+--------------+------------------+ +| 1 | 20 | 100 | +| 2 | 30 | NULL | ++------------+--------------+------------------+ +~~~ + +If the `DEFAULT` value constraint is not specified and an explicit value is not given, a value of *NULL* is assigned to the column. + +## See also + +- [Constraints](constraints.html) +- [`ALTER COLUMN`](alter-column.html) +- [`CHECK` constraint](check.html) +- [`REFERENCES` constraint (Foreign Key)](foreign-key.html) +- [`NOT NULL` constraint](not-null.html) +- [`PRIMARY KEY` constraint](primary-key.html) +- [`UNIQUE` constraint](unique.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) diff --git a/v20.2/delete-data.md b/v20.2/delete-data.md new file mode 100644 index 00000000000..a8dc6200e2f --- /dev/null +++ b/v20.2/delete-data.md @@ -0,0 +1,155 @@ +--- +title: Delete Data +summary: How to delete data from CockroachDB during application development +toc: true +--- + +This page has instructions for deleting data from CockroachDB (using the [`DELETE`](update.html) statement) using various programming languages. + +## Before you begin + +Make sure you have already: + +- Set up a [local cluster](secure-a-cluster.html). +- [Installed a Postgres client](install-client-drivers.html). +- [Connected to the database](connect-to-the-database.html). +- [Inserted data](insert-data.html) that you now want to delete. + +{% include {{page.version.version}}/app/retry-errors.md %} + +## Delete a single row + +
+ + + + +
+ +
+ +{% include copy-clipboard.html %} +~~~ sql +DELETE from accounts WHERE id = 1; +~~~ + +For more information about how to use the built-in SQL client, see the [`cockroach sql`](cockroach-sql.html) reference docs. + +
+ +
+ +{% include copy-clipboard.html %} +~~~ go +// 'db' is an open database connection + +if _, err := db.Exec("DELETE FROM accounts WHERE id = 1"); err != nil { + return err +} +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-go.md %} + +
+ +
+ +{% include copy-clipboard.html %} +~~~ java +// ds is an org.postgresql.ds.PGSimpleDataSource + +try (Connection connection = ds.getConnection()) { + connection.createStatement().executeUpdate("DELETE FROM accounts WHERE id = 1"); + +} catch (SQLException e) { + System.out.printf("sql state = [%s]\ncause = [%s]\nmessage = [%s]\n", + e.getSQLState(), e.getCause(), e.getMessage()); +} +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-java.md %} + +
+ +
+ +{% include copy-clipboard.html %} +~~~ python +# conn is a psycopg2 connection + +with conn.cursor() as cur: + cur.execute("DELETE FROM accounts WHERE id = 1", +conn.commit() +~~~ + +{% include {{page.version.version}}/app/for-a-complete-example-python.md %} + +
+ +## Delete multiple rows + +You can delete multiple rows from a table in several ways: + +- Using a `WHERE` clause to limit the number of rows based on one or more predicates: + + {% include copy-clipboard.html %} + ~~~ sql + DELETE FROM student_loan_accounts WHERE loan_amount < 30000; + ~~~ + +- Using a `WHERE` clause to specify multiple records by a specific column's value (in this case, `id`): + + {% include copy-clipboard.html %} + ~~~ sql + DELETE FROM accounts WHERE id IN (1, 2, 3, 4, 5); + ~~~ + +- Using [`TRUNCATE`](truncate.html) instead of [`DELETE`](delete.html) to delete all of the rows from a table, as recommended in our [performance best practices](performance-best-practices-overview.html#use-truncate-instead-of-delete-to-delete-all-rows-in-a-table). + +{{site.data.alerts.callout_info}} +Before deleting large amounts of data, see [Performance considerations](#performance-considerations). +{{site.data.alerts.end}} + +## Performance considerations + +Because of the way CockroachDB works under the hood, deleting data from the database does not immediately reduce disk usage. Instead, records are marked as "deleted" and processed asynchronously by a background garbage collection process. This process runs every 25 hours by default to allow sufficient time for running [backups](backup-and-restore.html) and running [time travel queries using `AS OF SYSTEM TIME`](as-of-system-time.html). The garbage collection interval is controlled by the [`gc.ttlseconds`](configure-replication-zones.html#replication-zone-variables) setting. + +The practical implications of the above are: + +- Deleting data will not immediately decrease disk usage. +- If you issue multiple [`DELETE`](delete.html) statements in sequence that each delete large amounts of data, each subsequent `DELETE` statement will run more slowly, for reasons [explained in this FAQ entry](sql-faqs.html#why-are-my-deletes-getting-slower-over-time). +- To delete all of the rows in a table, [it's faster to use `TRUNCATE` instead of `DELETE`](performance-best-practices-overview.html#use-truncate-instead-of-delete-to-delete-all-rows-in-a-table). + +For more information about how the storage layer of CockroachDB works, see the [storage layer reference documentation](architecture/storage-layer.html). + +## See also + +Reference information related to this task: + +- [`DELETE`](delete.html) +- [Disk space usage after deletes](delete.html#disk-space-usage-after-deletes) +- [Why are my deletes getting slower over time?](sql-faqs.html#why-are-my-deletes-getting-slower-over-time) +- [`TRUNCATE`](truncate.html) +- [`DROP TABLE`](drop-table.html) +- [Understanding and Avoiding Transaction Contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) +- [Delete Multiple Rows](delete.html#delete-specific-rows) + +Other common tasks: + +- [Connect to the Database](connect-to-the-database.html) +- [Insert Data](insert-data.html) +- [Query Data](query-data.html) +- [Update Data](update-data.html) +- [Run Multi-Statement Transactions](run-multi-statement-transactions.html) +- [Error Handling and Troubleshooting](error-handling-and-troubleshooting.html) +- [Make Queries Fast][fast] +- [Hello World Example apps](hello-world-example-apps.html) + + + +[selection]: selection-queries.html +[manual]: manual-deployment.html +[orchestrated]: orchestration.html +[fast]: make-queries-fast.html +[paginate]: selection-queries.html#paginate-through-limited-results +[joins]: joins.html diff --git a/v20.2/delete.md b/v20.2/delete.md new file mode 100644 index 00000000000..594d9ed0a05 --- /dev/null +++ b/v20.2/delete.md @@ -0,0 +1,408 @@ +--- +title: DELETE +summary: The DELETE statement deletes rows from a table. +toc: true +--- + +The `DELETE` [statement](sql-statements.html) deletes rows from a table. + +{{site.data.alerts.callout_danger}}If you delete a row that is referenced by a foreign key constraint and has an ON DELETE action, all of the dependent rows will also be deleted or updated.{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}}To delete columns, see DROP COLUMN.{{site.data.alerts.end}} + +## Required privileges + +The user must have the `DELETE` and `SELECT` [privileges](authorization.html#assign-privileges) on the table. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/delete.html %} +
+ +## Parameters + + + + Parameter | Description +-----------|------------- + `common_table_expr` | See [Common Table Expressions](common-table-expressions.html). + `table_name` | The name of the table that contains the rows you want to update. + `AS table_alias_name` | An alias for the table name. When an alias is provided, it completely hides the actual table name. +`WHERE a_expr`| `a_expr` must be an expression that returns Boolean values using columns (e.g., ` = `). Delete rows that return `TRUE`.

__Without a `WHERE` clause in your statement, `DELETE` removes all rows from the table.__ + `sort_clause` | An `ORDER BY` clause.

See [Ordering of rows in +DML statements](query-order.html#ordering-rows-in-dml-statements) for more details. + `limit_clause` | A `LIMIT` clause. See [Limiting Query Results](limit-offset.html) for more details. + `RETURNING target_list` | Return values based on rows deleted, where `target_list` can be specific column names from the table, `*` for all columns, or computations using [scalar expressions](scalar-expressions.html).

To return nothing in the response, not even the number of rows updated, use `RETURNING NOTHING`. + +## Success responses + +Successful `DELETE` statements return one of the following: + + Response | Description +-----------|------------- +`DELETE` _`int`_ | _int_ rows were deleted.

`DELETE` statements that do not delete any rows respond with `DELETE 0`. When `RETURNING NOTHING` is used, this information is not included in the response. +Retrieved table | Including the `RETURNING` clause retrieves the deleted rows, using the columns identified by the clause's parameters.

[See an example.](#return-deleted-rows) + +## Disk space usage after deletes + +Deleting a row does not immediately free up the disk space. This is +due to the fact that CockroachDB retains [the ability to query tables +historically](https://www.cockroachlabs.com/blog/time-travel-queries-select-witty_subtitle-the_future/). + +If disk usage is a concern, the solution is to +[reduce the time-to-live](configure-replication-zones.html) (TTL) for +the zone by setting `gc.ttlseconds` to a lower value, which will cause +garbage collection to clean up deleted objects (rows, tables) more +frequently. + +## Select performance on deleted rows + +Queries that scan across tables that have lots of deleted rows will +have to scan over deletions that have not yet been garbage +collected. Certain database usage patterns that frequently scan over +and delete lots of rows will want to reduce the +[time-to-live](configure-replication-zones.html) values to clean up +deleted rows more frequently. + +## Sorting the output of deletes + +{% include {{page.version.version}}/misc/sorting-delete-output.md %} + +For more information about ordering query results in general, see +[Ordering Query Results](query-order.html) and [Ordering of rows in +DML statements](query-order.html#ordering-rows-in-dml-statements). + +## Delete performance on large data sets + +If you are deleting a large amount of data using iterative `DELETE ... LIMIT` statements, you are likely to see a drop in performance for each subsequent `DELETE` statement. + +For an explanation of why this happens, and for instructions showing how to iteratively delete rows in constant time, see [Why are my deletes getting slower over time?](sql-faqs.html#why-are-my-deletes-getting-slower-over-time). + +## Force index selection for deletes + +By using the explicit index annotation (also known as "index hinting"), you can override [CockroachDB's index selection](https://www.cockroachlabs.com/blog/index-selection-cockroachdb-2/) and use a specific [index](indexes.html) for deleting rows of a named table. + +{{site.data.alerts.callout_info}} +Index selection can impact [performance](performance-best-practices-overview.html), but does not change the result of a query. +{{site.data.alerts.end}} + +The syntax to force a specific index for a delete is: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM table@my_idx; +~~~ + +This is equivalent to the longer expression: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM table@{FORCE_INDEX=my_idx}; +~~~ + +To view how the index hint modifies the query plan that CockroachDB follows for deleting rows, use an [`EXPLAIN`](explain.html#opt-option) statement. To see all indexes available on a table, use [`SHOW INDEXES`](show-index.html). + +For examples, see [Delete with index hints](#delete-with-index-hints). + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Delete all rows + +You can delete all rows from a table by not including a `WHERE` clause in your `DELETE` statement. + +{{site.data.alerts.callout_info}} +If the [`sql_safe_updates`](cockroach-sql.html#allow-potentially-unsafe-sql-statements) session variable is set to `true`, the client will prevent the update. `sql_safe_updates` is set to `true` by default. +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM vehicle_location_histories; +~~~ + +~~~ +pq: rejected: DELETE without WHERE clause (sql_safe_updates = true) +~~~ + +You can use a [`SET`](set-vars.html) statement to set session variables. + +{% include copy-clipboard.html %} +~~~ sql +> SET sql_safe_updates = false; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM vehicle_location_histories; +~~~ + +~~~ +DELETE 1000 +~~~ + +{{site.data.alerts.callout_success}} +Unless your table is small (less than 1000 rows), using [`TRUNCATE`][truncate] to delete the contents of a table will be more performant than using `DELETE`. +{{site.data.alerts.end}} + +### Delete specific rows + +When deleting specific rows from a table, the most important decision you make is which columns to use in your `WHERE` clause. When making that choice, consider the potential impact of using columns with the [Primary Key](primary-key.html)/[Unique](unique.html) constraints (both of which enforce uniqueness) versus those that are not unique. + +#### Delete rows using Primary Key/unique columns + +Using columns with the [Primary Key](primary-key.html) or [Unique](unique.html) constraints to delete rows ensures your statement is unambiguous—no two rows contain the same column value, so it's less likely to delete data unintentionally. + +In this example, `code` is our primary key and we want to delete the row where the code equals "about_stuff_city". Because we're positive no other rows have that value in the `code` column, there's no risk of accidentally removing another row. + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM promo_codes WHERE code = 'about_stuff_city'; +~~~ +~~~ +DELETE 1 +~~~ + +#### Delete rows using non-unique columns + +Deleting rows using non-unique columns removes _every_ row that returns `TRUE` for the `WHERE` clause's `a_expr`. This can easily result in deleting data you didn't intend to. + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM promo_codes WHERE creation_time > '2019-01-30 00:00:00+00:00'; +~~~ +~~~ +DELETE 4 +~~~ + +The example statement deleted four rows, which might be unexpected. + +### Return deleted rows + +To see which rows your statement deleted, include the `RETURNING` clause to retrieve them using the columns you specify. + +#### Use all columns + +By specifying `*`, you retrieve all columns of the delete rows. + +#### Use specific columns + +To retrieve specific columns, name them in the `RETURNING` clause. + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM promo_codes WHERE creation_time > '2019-01-29 00:00:00+00:00' RETURNING code, rules; +~~~ +~~~ + code | rules ++------------------------+----------------------------------------------+ + box_investment_stuff | {"type": "percent_discount", "value": "10%"} + energy_newspaper_field | {"type": "percent_discount", "value": "10%"} + simple_guy_theory | {"type": "percent_discount", "value": "10%"} + study_piece_war | {"type": "percent_discount", "value": "10%"} + tv_this_list | {"type": "percent_discount", "value": "10%"} +(5 rows) + +~~~ + +#### Change column labels + +When `RETURNING` specific columns, you can change their labels using `AS`. + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM promo_codes WHERE creation_time > '2019-01-28 00:00:00+00:00' RETURNING code, rules AS discount; +~~~ +~~~ + code | discount ++---------------------+----------------------------------------------+ + chair_company_state | {"type": "percent_discount", "value": "10%"} + view_reveal_radio | {"type": "percent_discount", "value": "10%"} +(2 rows) +~~~ + +#### Sort and return deleted rows + +To sort and return deleted rows, use a statement like the following: + +{% include copy-clipboard.html %} +~~~ sql +> WITH a AS (DELETE FROM promo_codes WHERE creation_time > '2019-01-27 00:00:00+00:00' RETURNING *) + SELECT * FROM a ORDER BY expiration_time; +~~~ + +~~~ + code | description | creation_time | expiration_time | rules ++----------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------+---------------------------+----------------------------------------------+ + often_thing_hair | Society right wish face see if pull. Great generation social bar read budget wonder natural. Somebody dark field economic material. Nature nature paper law worry common. Serious activity hospital wide none. | 2019-01-27 03:04:05+00:00 | 2019-01-29 03:04:05+00:00 | {"type": "percent_discount", "value": "10%"} + step_though_military | Director middle summer most create any. | 2019-01-27 03:04:05+00:00 | 2019-01-29 03:04:05+00:00 | {"type": "percent_discount", "value": "10%"} + own_whose_economy | Social participant order this. Guy toward nor indeed police player inside nor. Model education voice several college art on. Start listen their maybe. | 2019-01-27 03:04:05+00:00 | 2019-01-30 03:04:05+00:00 | {"type": "percent_discount", "value": "10%"} + crime_experience_certainly | Prepare right teacher mouth student. Trouble condition weight during scene something stand. | 2019-01-27 03:04:05+00:00 | 2019-01-31 03:04:05+00:00 | {"type": "percent_discount", "value": "10%"} + policy_its_wife | Player either she something good minute or. Nearly policy player receive. Somebody mean book store fire realize. | 2019-01-27 03:04:05+00:00 | 2019-01-31 03:04:05+00:00 | {"type": "percent_discount", "value": "10%"} +(5 rows) +~~~ + +### Delete with index hints + +Suppose you create a multi-column index on the `users` table with the `name` and `city` columns. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX ON users (name, city); +~~~ + +Now suppose you want to delete the two users named "Jon Snow". You can use the [`EXPLAIN (OPT)`](explain.html#opt-option) command to see how the [cost-based optimizer](cost-based-optimizer.html) decides to perform the delete: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT) DELETE FROM users WHERE name='Jon Snow'; +~~~ + +~~~ + text +------------------------------------------------------------------------------- + delete users + ├── scan users@users_name_city_idx + │ └── constraint: /8/7/6: [/'Jon Snow' - /'Jon Snow'] + └── f-k-checks + ├── f-k-checks-item: vehicles(city,owner_id) -> users(city,id) + │ └── semi-join (hash) + │ ├── with-scan &1 + │ ├── scan vehicles@vehicles_auto_index_fk_city_ref_users + │ └── filters + │ ├── city = vehicles.city + │ └── id = owner_id + ├── f-k-checks-item: rides(city,rider_id) -> users(city,id) + │ └── semi-join (lookup rides@rides_auto_index_fk_city_ref_users) + │ ├── with-scan &1 + │ └── filters (true) + └── f-k-checks-item: user_promo_codes(city,user_id) -> users(city,id) + └── semi-join (hash) + ├── with-scan &1 + ├── scan user_promo_codes + └── filters + ├── city = user_promo_codes.city + └── id = user_id +(22 rows) +~~~ + +The output of the `EXPLAIN` statement shows that the optimizer scans the newly-created `users_name_city_idx` index when performing the delete. This makes sense, as you are performing a delete based on the `name` column. + +Now suppose that instead you want to perform a delete, but using the `id` column instead. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT) DELETE FROM users WHERE id IN ('70a3d70a-3d70-4400-8000-000000000016', '3d70a3d7-0a3d-4000-8000-00000000000c'); +~~~ + +~~~ + delete users + ├── select + │ ├── scan users@users_name_city_idx + │ └── filters + │ └── users.id IN ('3d70a3d7-0a3d-4000-8000-00000000000c', '70a3d70a-3d70-4400-8000-000000000016') + └── f-k-checks + ├── f-k-checks-item: vehicles(city,owner_id) -> users(city,id) + │ └── semi-join (hash) + │ ├── with-scan &1 + │ ├── scan vehicles@vehicles_auto_index_fk_city_ref_users + │ └── filters + │ ├── city = vehicles.city + │ └── id = owner_id + ├── f-k-checks-item: rides(city,rider_id) -> users(city,id) + │ └── semi-join (lookup rides@rides_auto_index_fk_city_ref_users) + │ ├── with-scan &1 + │ └── filters (true) + └── f-k-checks-item: user_promo_codes(city,user_id) -> users(city,id) + └── semi-join (hash) + ├── with-scan &1 + ├── scan user_promo_codes + └── filters + ├── city = user_promo_codes.city + └── id = user_id +(24 rows) +~~~ + +The optimizer still scans the newly-created `users_name_city_idx` index when performing the delete. Although scanning the table on this index could still be the most efficient, you may want to assess the performance difference between using `users_name_city_idx` and an index on the `id` column, as you are performing a delete with a filter on the `id` column. + +If you provide an index hint (i.e. force the index selection) to use the primary index on the column instead, the CockroachDB will scan the users table using the primary index, on `city`, and `id`. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT) DELETE FROM users@primary WHERE id IN ('70a3d70a-3d70-4400-8000-000000000016', '3d70a3d7-0a3d-4000-8000-00000000000c'); +~~~ + +~~~ + text +--------------------------------------------------------------------------------------------------------------- + delete users + ├── select + │ ├── scan users + │ │ └── flags: force-index=primary + │ └── filters + │ └── users.id IN ('3d70a3d7-0a3d-4000-8000-00000000000c', '70a3d70a-3d70-4400-8000-000000000016') + └── f-k-checks + ├── f-k-checks-item: vehicles(city,owner_id) -> users(city,id) + │ └── semi-join (hash) + │ ├── with-scan &1 + │ ├── scan vehicles@vehicles_auto_index_fk_city_ref_users + │ └── filters + │ ├── city = vehicles.city + │ └── id = owner_id + ├── f-k-checks-item: rides(city,rider_id) -> users(city,id) + │ └── semi-join (lookup rides@rides_auto_index_fk_city_ref_users) + │ ├── with-scan &1 + │ └── filters (true) + └── f-k-checks-item: user_promo_codes(city,user_id) -> users(city,id) + └── semi-join (hash) + ├── with-scan &1 + ├── scan user_promo_codes + └── filters + ├── city = user_promo_codes.city + └── id = user_id +(25 rows) +~~~ + +## See also + +- [`INSERT`](insert.html) +- [`UPDATE`](update.html) +- [`UPSERT`](upsert.html) +- [`TRUNCATE`][truncate] +- [`ALTER TABLE`](alter-table.html) +- [`DROP TABLE`](drop-table.html) +- [`DROP DATABASE`](drop-database.html) +- [Other SQL Statements](sql-statements.html) +- [Limiting Query Results](limit-offset.html) + + + +[truncate]: truncate.html + + diff --git a/v20.2/demo-automatic-cloud-migration.md b/v20.2/demo-automatic-cloud-migration.md new file mode 100644 index 00000000000..ea75baf9ca8 --- /dev/null +++ b/v20.2/demo-automatic-cloud-migration.md @@ -0,0 +1,274 @@ +--- +title: Cross-Cloud Migration +summary: Use a local cluster to simulate migrating from one cloud platform to another. +toc: true +--- + +CockroachDB's flexible [replication controls](configure-replication-zones.html) make it trivially easy to run a single CockroachDB cluster across cloud platforms and to migrate data from one cloud to another without any service interruption. This page walks you through a local simulation of the process. + +## Watch the demo + + + +## Step 1. Install prerequisites + +In this tutorial, you'll use CockroachDB, its built-in `ycsb` workload, and the HAProxy load balancer. Before you begin, make sure these applications are installed: + +- Install the latest version of [CockroachDB](install-cockroachdb.html). +- Install [HAProxy](http://www.haproxy.org/). If you're on a Mac and using Homebrew, use `brew install haproxy`. + +Also, to keep track of the data files and logs for your cluster, you may want to create a new directory (e.g., `mkdir cloud-migration`) and start all your nodes in that directory. + +## Step 2. Start a 3-node cluster on "cloud 1" + +If you've already [started a local cluster](start-a-local-cluster.html), the commands for starting nodes should be familiar to you. The new flag to note is [`--locality`](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes), which accepts key-value pairs that describe the topography of a node. In this case, you're using the flag to specify that the first 3 nodes are running on cloud 1. + +In a new terminal, start node 1 on cloud 1: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=1 \ +--store=cloud1node1 \ +--listen-addr=localhost:26257 \ +--http-addr=localhost:8080 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~~ + +In a new terminal, start node 2 on cloud 1: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=1 \ +--store=cloud1node2 \ +--listen-addr=localhost:26258 \ +--http-addr=localhost:8081 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~ + +In a new terminal, start node 3 on cloud 1: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=1 \ +--store=cloud1node3 \ +--listen-addr=localhost:26259 \ +--http-addr=localhost:8082 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~ + +## Step 3. Initialize the cluster + +In a new terminal, use the [`cockroach init`](cockroach-init.html) command to perform a one-time initialization of the cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init \ +--insecure \ +--host=localhost:26257 +~~~ + +## Step 4. Set up HAProxy load balancing + +You're now running 3 nodes in a simulated cloud. Each of these nodes is an equally suitable SQL gateway to your cluster, but to ensure an even balancing of client requests across these nodes, you can use a TCP load balancer. Let's use the open-source [HAProxy](http://www.haproxy.org/) load balancer that you installed earlier. + +In a new terminal, run the [`cockroach gen haproxy`](cockroach-gen.html) command, specifying the port of any node: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen haproxy \ +--insecure \ +--host=localhost:26257 +~~~ + +This command generates an `haproxy.cfg` file automatically configured to work with the 3 nodes of your running cluster. In the file, change `bind :26257` to `bind :26000`. This changes the port on which HAProxy accepts requests to a port that is not already in use by a node and that will not be used by the nodes you'll add later. + +~~~ +global + maxconn 4096 + +defaults + mode tcp + # Timeout values should be configured for your specific use. + # See: https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20connect + timeout connect 10s + timeout client 1m + timeout server 1m + # TCP keep-alive on client side. Server already enables them. + option clitcpka + +listen psql + bind :26000 + mode tcp + balance roundrobin + option httpchk GET /health?ready=1 + server cockroach1 localhost:26257 check port 8080 + server cockroach2 localhost:26258 check port 8081 + server cockroach3 localhost:26259 check port 8082 +~~~ + +Start HAProxy, with the `-f` flag pointing to the `haproxy.cfg` file: + +{% include copy-clipboard.html %} +~~~ shell +$ haproxy -f haproxy.cfg +~~~ + +## Step 5. Run a sample workload + +Now that you have a load balancer running in front of your cluster, lets use the YCSB workload built into CockroachDB to simulate multiple client connections, each performing mixed read/write workloads. + +1. In a new terminal, load the initial `ycsb` schema and data, pointing it at HAProxy's port: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init ycsb \ + 'postgresql://root@localhost:26000?sslmode=disable' + ~~~ + +2. Run the `ycsb` workload, pointing it at HAProxy's port: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run ycsb \ + --duration=20m \ + --concurrency=10 \ + --max-rate=1000 + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + This command initiates 10 concurrent client workloads for 20 minutes, but limits the total load to 1000 operations per second (since you're running everything on a single machine). + + You'll soon see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1s 0 9258.1 9666.6 0.7 1.3 2.0 8.9 read + 1s 0 470.1 490.9 1.7 2.9 4.1 5.0 update + 2s 0 10244.6 9955.6 0.7 1.2 2.0 6.6 read + 2s 0 559.0 525.0 1.6 3.1 6.0 7.3 update + 3s 0 9870.8 9927.4 0.7 1.4 2.4 10.0 read + 3s 0 500.0 516.6 1.6 4.2 7.9 15.2 update + 4s 0 9847.2 9907.3 0.7 1.4 2.4 23.1 read + 4s 0 506.8 514.2 1.6 3.7 7.6 17.8 update + 5s 0 10084.4 9942.6 0.7 1.3 2.1 7.1 read + 5s 0 537.2 518.8 1.5 3.5 10.0 15.2 update + ... + ~~~ + +## Step 6. Watch data balance across all 3 nodes + +Now open the Admin UI at `http://localhost:8080` and click **Metrics** in the left-hand navigation bar. The **Overview** dashboard is displayed. Hover over the **SQL Queries** graph at the top. After a minute or so, you'll see that the load generator is executing approximately 95% reads and 5% writes across all nodes: + +CockroachDB Admin UI + +Scroll down a bit and hover over the **Replicas per Node** graph. Because CockroachDB replicates each piece of data 3 times by default, the replica count on each of your 3 nodes should be identical: + +CockroachDB Admin UI + +## Step 7. Add 3 nodes on "cloud 2" + +At this point, you're running three nodes on cloud 1. But what if you'd like to start experimenting with resources provided by another cloud vendor? Let's try that by adding three more nodes to a new cloud platform. Again, the flag to note is [`--locality`](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes), which you're using to specify that these next 3 nodes are running on cloud 2. + +In a new terminal, start node 4 on cloud 2: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=2 \ +--store=cloud2node4 \ +--listen-addr=localhost:26260 \ +--http-addr=localhost:8083 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~ + +In a new terminal, start node 5 on cloud 2: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=2 \ +--store=cloud2node5 \ +--advertise-addr=localhost:26261 \ +--http-addr=localhost:8084 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~ + +In a new terminal, start node 6 on cloud 2: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--locality=cloud=2 \ +--store=cloud2node6 \ +--advertise-addr=localhost:26262 \ +--http-addr=localhost:8085 \ +--cache=100MB \ +--join=localhost:26257,localhost:26258,localhost:26259 +~~~ + +## Step 8. Watch data balance across all 6 nodes + +Back on the **Overview** dashboard in Admin UI, hover over the **Replicas per Node** graph again. Because you used [`--locality`](configure-replication-zones.html#descriptive-attributes-assigned-to-nodes) to specify that nodes are running on 2 clouds, you'll see an approximately even number of replicas on each node, indicating that CockroachDB has automatically rebalanced replicas across both simulated clouds: + +CockroachDB Admin UI + +Note that it takes a few minutes for the Admin UI to show accurate per-node replica counts on hover. This is why the new nodes in the screenshot above show 0 replicas. However, the graph lines are accurate, and you can click **View node list** in the **Summary** area for accurate per-node replica counts as well. + +## Step 9. Migrate all data to "cloud 2" + +So your cluster is replicating across two simulated clouds. But let's say that after experimentation, you're happy with cloud vendor 2, and you decide that you'd like to move everything there. Can you do that without interruption to your live client traffic? Yes, and it's as simple as running a single command to add a [hard constraint](configure-replication-zones.html#replication-constraints) that all replicas must be on nodes with `--locality=cloud=2`. + +In a new terminal, [edit the default replication zone](configure-zone.html): + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --execute="ALTER RANGE default CONFIGURE ZONE USING constraints='[+cloud=2]';" --insecure --host=localhost:26257 +~~~ + +## Step 10. Verify the data migration + +Back on the **Overview** dashboard in the Admin UI, hover over the **Replicas per Node** graph again. Very soon, you'll see the replica count double on nodes 4, 5, and 6 and drop to 0 on nodes 1, 2, and 3: + +CockroachDB Admin UI + +This indicates that all data has been migrated from cloud 1 to cloud 2. In a real cloud migration scenario, at this point you would update the load balancer to point to the nodes on cloud 2 and then stop the nodes on cloud 1. But for the purpose of this local simulation, there's no need to do that. + +## Step 11. Stop the cluster + +Once you're done with your cluster, stop YCSB by switching into its terminal and pressing **CTRL-C**. Then do the same for HAProxy and each CockroachDB node. + +{{site.data.alerts.callout_success}}For the last node, the shutdown process will take longer (about a minute) and will eventually force kill the node. This is because, with only 1 node still online, a majority of replicas are no longer available (2 of 3), and so the cluster is not operational. To speed up the process, press CTRL-C a second time.{{site.data.alerts.end}} + +If you do not plan to restart the cluster, you may want to remove the nodes' data stores and the HAProxy config file: + +{% include copy-clipboard.html %} +~~~ shell +$ rm -rf cloud1node1 cloud1node2 cloud1node3 cloud2node4 cloud2node5 cloud2node6 haproxy.cfg +~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} + +You may also want to learn other ways to control the location and number of replicas in a cluster: + +- [Even Replication Across Datacenters](configure-replication-zones.html#even-replication-across-datacenters) +- [Multiple Applications Writing to Different Databases](configure-replication-zones.html#multiple-applications-writing-to-different-databases) +- [Stricter Replication for a Table and Its Indexes](configure-replication-zones.html#stricter-replication-for-a-table-and-its-secondary-indexes) +- [Tweaking the Replication of System Ranges](configure-replication-zones.html#tweaking-the-replication-of-system-ranges) diff --git a/v20.2/demo-fault-tolerance-and-recovery.md b/v20.2/demo-fault-tolerance-and-recovery.md new file mode 100644 index 00000000000..3c9083ccf64 --- /dev/null +++ b/v20.2/demo-fault-tolerance-and-recovery.md @@ -0,0 +1,401 @@ +--- +title: Fault Tolerance & Recovery +summary: Use a local cluster to explore how CockroachDB remains available during, and recovers after, failure. +toc: true +--- + +This page walks you through a simple demonstration of how CockroachDB remains available during, and recovers after, failure. Starting with a 6-node local cluster with the default 3-way replication, you'll run a sample workload, stop a node to simulate failure, and see how the cluster continues uninterrupted. You'll then leave that node offline for long enough to watch the cluster repair itself by re-replicating missing data to other nodes. You'll then prepare the cluster for 2 simultaneous node failures by increasing to 5-way replication, then take two nodes offline at the same time, and again see how the cluster continues uninterrupted. + +## Before you begin + +Make sure you have already [installed CockroachDB](install-cockroachdb.html). + +## Step 1. Start a 6-node cluster + +1. Use the [`cockroach start`](cockroach-start.html) command to start 6 nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node1 \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node2 \ + --listen-addr=localhost:26258 \ + --http-addr=localhost:8081 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node3 \ + --listen-addr=localhost:26259 \ + --http-addr=localhost:8082 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node4 \ + --listen-addr=localhost:26260 \ + --http-addr=localhost:8083 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node5 \ + --listen-addr=localhost:26261 \ + --http-addr=localhost:8084 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node6 \ + --listen-addr=localhost:26262 \ + --http-addr=localhost:8085 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +2. Use the [`cockroach init`](cockroach-init.html) command to perform a one-time initialization of the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init \ + --insecure \ + --host=localhost:26257 + ~~~ + +## Step 2. Set up load balancing + +In this module, you'll run a sample workload to simulate multiple client connections. Each node is an equally suitable SQL gateway for the load, but it's always recommended to [spread requests evenly across nodes](recommended-production-settings.html#load-balancing). You'll use the open-source [HAProxy](http://www.haproxy.org/) load balancer to do that here. + +1. Install HAProxy. + +
+ + +
+

+ +
+ If you're on a Mac and use Homebrew, run: + {% include copy-clipboard.html %} + ~~~ shell + $ brew install haproxy + ~~~ +
+ +
+ If you're using Linux and use apt-get, run: + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get install haproxy + ~~~ +
+ +2. Run the [`cockroach gen haproxy`](../cockroach-gen.html) command, specifying the port of any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach gen haproxy \ + --insecure \ + --host=localhost \ + --port=26257 + ~~~ + + This command generates an `haproxy.cfg` file automatically configured to work with the nodes of your running cluster. + +3. In `haproxy.cfg`, change `bind :26257` to `bind :26000`. This changes the port on which HAProxy accepts requests to a port that is not already in use by a node. + + {% include copy-clipboard.html %} + ~~~ shell + sed -i.saved 's/^ bind :26257/ bind :26000/' haproxy.cfg + ~~~ + +4. Start HAProxy, with the `-f` flag pointing to the `haproxy.cfg` file: + + {% include copy-clipboard.html %} + ~~~ shell + $ haproxy -f haproxy.cfg & + ~~~ + +## Step 3. Run a sample workload + +Now that you have a load balancer running in front of your cluster, use the [`cockroach workload`](cockroach-workload.html) command to run CockroachDB's built-in version of the YCSB benchmark, simulating multiple client connections, each performing mixed read/write operations. + +1. Load the initial `ycsb` schema and data, pointing it at HAProxy's port: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init ycsb --splits=50 \ + 'postgresql://root@localhost:26000?sslmode=disable' + ~~~ + + The `--splits` flag tells the workload to manually split ranges a number of times. This is not something you'd normally do, but for the purpose of this tutorial, it makes it easier to visualize the movement of data in the cluster. + +2. Run the `ycsb` workload, pointing it at HAProxy's port: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run ycsb \ + --duration=20m \ + --concurrency=3 \ + --max-rate=1000 \ + --tolerate-errors \ + 'postgresql://root@localhost:26000?sslmode=disable' + ~~~ + + This command initiates 3 concurrent client workloads for 20 minutes, but limits the total load to 1000 operations per second (since you're running everything on a single machine). + + You'll see per-operation statistics print to standard output every second: + + ~~~ + _elapsed___errors__ops/sec(inst)___ops/sec(cum)__p50(ms)__p95(ms)__p99(ms)_pMax(ms) + 1.0s 0 902.8 930.9 1.1 2.1 4.1 62.9 read + 1.0s 0 46.5 48.0 3.4 4.7 6.0 6.0 update + 2.0s 0 923.3 926.9 1.0 2.9 5.5 8.9 read + 2.0s 0 38.0 43.0 3.0 6.0 7.6 7.6 update + 3.0s 0 901.1 918.3 1.1 2.5 5.0 6.6 read + 3.0s 0 55.0 47.0 3.4 7.9 9.4 11.5 update + 4.0s 0 948.9 926.0 1.0 1.6 2.6 5.0 read + 4.0s 0 46.0 46.7 3.1 5.2 16.8 16.8 update + 5.0s 0 932.0 927.2 1.1 1.8 2.9 13.6 read + 5.0s 0 56.0 48.6 3.0 4.2 5.2 5.5 update + ... + ~~~ + + After the specified duration (20 minutes in this case), the workload will stop and you'll see totals printed to standard output. + +## Step 4. Check the workload + +Initially, the workload creates a new database called `ycsb`, creates a `usertable` table in that database, and inserts a bunch of rows into the table. Soon, the load generator starts executing approximately 95% reads and 5% writes. + +1. Go to the Admin UI at http://localhost:8080. + +2. To check the SQL queries getting executed, click **Metrics** on the left, and hover over the **SQL Queries** graph at the top: + + CockroachDB Admin UI + +3. To check the client connections from the load generator, select the **SQL** dashboard and hover over the **SQL Connections** graph: + + CockroachDB Admin UI + + You'll notice 3 client connections from the load generator. If you want to check that HAProxy balanced each connection to a different node, you can change the **Graph** dropdown from **Cluster** to specific nodes. + +4. To see more details about the `ycsb` database and `usertable` table, click **Databases** in the upper left and then scroll down until you see **ycsb**: + + CockroachDB Admin UI + + You can also view the schema of the `usertable` by clicking the table name: + + CockroachDB Admin UI + +5. By default, CockroachDB replicates all data 3 times and balances it across all nodes. To see this balance, click **Overview** and check the replica count across all nodes: + + CockroachDB Admin UI + +## Step 5. Simulate a single node failure + +When a node fails, the cluster waits for the node to remain offline for 5 minutes by default before considering it dead, at which point the cluster automatically repairs itself by re-replicating any of the replicas on the down nodes to other available nodes. + +1. In a new terminal, [edit the default replication zone](configure-replication-zones.html) to reduce the amount of time the cluster waits before considering a node dead to the minimum allowed of 1 minute and 15 seconds: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --insecure \ + --host=localhost:26000 \ + --execute="SET CLUSTER SETTING server.time_until_store_dead = '1m15s';" + ~~~ + +2. Then use the [`cockroach quit`](../cockroach-quit.html) command to stop a node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit \ + --insecure \ + --host=localhost:26261 + ~~~ + +## Step 6. Check load continuity and cluster health + +Go back to the Admin UI, click **Metrics** on the left, and verify that the cluster as a whole continues serving data, despite one of the nodes being unavailable and marked as **Suspect**: + +CockroachDB Admin UI + +This shows that when all ranges are replicated 3 times (the default), the cluster can tolerate a single node failure because the surviving nodes have a majority of each range's replicas (2/3). + +## Step 7. Watch the cluster repair itself + +Click **Overview** on the left: + +CockroachDB Admin UI + +Because you reduced the time it takes for the cluster to consider the down node dead, after 1 minute or so, the cluster will consider the down node "dead", and you'll see the replica count on the remaining nodes increase and the number of under-replicated ranges decrease to 0. This shows the cluster repairing itself by re-replicating missing replicas. + +## Step 8. Prepare for two simultaneous node failures + +At this point, the cluster has recovered and is ready to handle another failure. However, the cluster cannot handle two _near-simultaneous_ failures in this configuration. Failures are "near-simultaneous" if they are closer together than the `server.time_until_store_dead` [cluster setting](cluster-settings.html) plus the time taken for the number of replicas on the dead node to drop to zero. If two failures occurred in this configuration, some ranges would become unavailable until one of the nodes recovers. + +To be able to tolerate 2 of 5 nodes failing simultaneously without any service interruption, ranges must be replicated 5 times. + +1. Restart the dead node, using the same command you used to start the node initially: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=fault-node5 \ + --listen-addr=localhost:26261 \ + --http-addr=localhost:8084 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +2. Use the [`ALTER RANGE ... CONFIGURE ZONE`](../configure-zone.html) command to change the cluster's `default` replication factor to 5: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --execute="ALTER RANGE default CONFIGURE ZONE USING num_replicas=5;" --insecure --host=localhost:26000 + ~~~ + +3. Back in the Admin UI **Overview** dashboard, watch the replica count increases and even out across all 6 nodes: + + CockroachDB Admin UI + + This shows the cluster up-replicating so that each range has 5 replicas, one on each node. + +## Step 9. Simulate two simultaneous node failures + +Use the [`cockroach quit`](../cockroach-quit.html) command to stop two nodes: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --insecure --host=localhost:26260 +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --insecure --host=localhost:26261 +~~~ + +## Step 10. Check load continuity and cluster health + +1. Like before, go to the Admin UI, click **Metrics** on the left, and verify that the cluster as a whole continues serving data, despite 2 nodes being offline: + + CockroachDB Admin UI + + This shows that when all ranges are replicated 5 times, the cluster can tolerate 2 simultaneous node outages because the surviving nodes have a majority of each range's replicas (3/5). + +2. To verify this further, use the `cockroach sql` command to count the number of rows in the `ycsb.usertable` table and verify that it is still serving reads: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --insecure \ + --host=localhost:26257 \ + --execute="SELECT count(*) FROM ycsb.usertable;" + ~~~ + + ~~~ + count + +-------+ + 10000 + (1 row) + ~~~ + + And writes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --insecure \ + --host=localhost:26257 \ + --execute="INSERT INTO ycsb.usertable VALUES ('asdf', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);" + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --insecure \ + --host=localhost:26257 \ + --execute="SELECT count(*) FROM ycsb.usertable;" + ~~~ + + ~~~ + count + +-------+ + 10001 + (1 row) + ~~~ + +## Step 11. Clean up + +1. In the terminal where the YCSB workload is running, press CTRL + c. + +2. Stop HAProxy: + + {% include copy-clipboard.html %} + ~~~ shell + $ pkill haproxy + ~~~ + +3. Use the [`cockroach quit`](cockroach-quit.html) command to shut down the remaining 4 nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26257 + ~~~ + + {{site.data.alerts.callout_info}} + For the final 2 nodes, the shutdown process will take longer (about a minute each) and will eventually force the nodes to stop. This is because, with only 2 of 5 nodes left, a majority of replicas are not available, and so the cluster is no longer operational. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26258 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26259 + ~~~ + +4. To restart the cluster at a later time, run the same `cockroach start` commands as earlier from the directory containing the nodes' data stores. + + If you do not plan to restart the cluster, you may want to remove the nodes' data stores and the HAProxy config files: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm -rf fault-node1 fault-node2 fault-node3 fault-node4 fault-node5 fault-node6 haproxy.cfg haproxy.cfg.saved + ~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} diff --git a/v20.2/demo-follow-the-workload.md b/v20.2/demo-follow-the-workload.md new file mode 100644 index 00000000000..c120acc3e07 --- /dev/null +++ b/v20.2/demo-follow-the-workload.md @@ -0,0 +1,286 @@ +--- +title: Follow-the-Workload +summary: CockroachDB can dynamically optimize read latency for the location from which most of the workload is originating. +toc: true +--- + +"Follow-the-workload" refers to CockroachDB's ability to dynamically optimize read latency for the location from which most of the workload is originating. This page explains how "follow-the-workload" works and walks you through a simple demonstration using a local cluster. + +{{site.data.alerts.callout_info}} +In practice, follow-the-workload is most useful when a CockroachDB cluster is running across multiple regions, with high latency between them, but other patterns are often preferable. For more details, see [Multi-Region Topology Patterns](topology-patterns.html#multi-region-patterns). +{{site.data.alerts.end}} + +## Before you begin + +### Basic concepts + +To understand how "follow-the-workload" works, it's important to start with some basic concepts: + +Concept | Description +--------|------------ +**Range** | CockroachDB stores all user data (tables, indexes, etc.) and almost all system data in a giant sorted map of key-value pairs. This keyspace is divided into "ranges", contiguous chunks of the keyspace, so that every key can always be found in a single range. +**Replica** | CockroachDB replicates each range (3 times by default) and stores each replica on a different node. +**Leaseholder** | For each range, one of the replicas holds the "range lease". This replica, referred to as the "leaseholder", is the one that receives and coordinates all read and write requests for the range.

Unlike writes, read requests access the leaseholder and send the results to the client without needing to coordinate with any of the other range replicas. This reduces the network round trips involved and is possible because the leaseholder is guaranteed to be up-to-date due to the fact that all write requests also go to the leaseholder. + +### How it works + +"Follow-the-workload" is based on the way **range leases** handle read requests. Read requests bypass the Raft consensus protocol, accessing the range replica that holds the range lease (the leaseholder) and sending the results to the client without needing to coordinate with any of the other range replicas. Bypassing Raft, and the network round trips involved, is possible because the leaseholder is guaranteed to be up-to-date due to the fact that all write requests also go to the leaseholder. + +This increases the speed of reads, but it doesn't guarantee that the range lease will be anywhere close to the origin of requests. If requests are coming from the US West, for example, and the relevant range lease is on a node in the US East, the requests would likely enter a gateway node in the US West and then get routed to the node with the range lease in the US East. + +However, you can cause the cluster to actively move range leases for even better read performance by starting each node with the [`--locality`](cockroach-start.html#locality) flag. With this flag specified, the cluster knows about the location of each node, so when there's high latency between nodes, the cluster will move active range leases to a node closer to the origin of the majority of the workload. This is especially helpful for applications with workloads that move around throughout the day (e.g., most of the traffic is in the US East in the morning and in the US West in the evening). + +{{site.data.alerts.callout_success}} +To enable "follow-the-workload", you just need to start each node of the cluster with the `--locality` flag, as shown in the tutorial below. No additional user action is required. +{{site.data.alerts.end}} + +### Example + +In this example, let's imagine that lots of read requests are going to node 1, and that the requests are for data in range 3. Because range 3's lease is on node 3, the requests are routed to node 3, which returns the results to node 1. Node 1 then responds to the clients. + +Follow-the-workload example + +However, if the nodes were started with the [`--locality`](cockroach-start.html#locality) flag, after a short while, the cluster would move range 3's lease to node 1, which is closer to the origin of the workload, thus reducing the network round trips and increasing the speed of reads. + +Follow-the-workload example + +## Step 1. Install prerequisites + +In this tutorial, you'll use CockroachDB, the `comcast` network tool to simulate network latency on your local workstation, and the `tpcc` workload built into CockroachDB to simulate client workloads. Before you begin, make sure these applications are installed: + +- Install the latest version of [CockroachDB](install-cockroachdb.html). +- Install [Go](https://golang.org/doc/install) version 1.9 or higher. If you're on a Mac and using Homebrew, use `brew install go`. You can check your local version by running `go version`. +- Install the [`comcast`](https://github.com/tylertreat/comcast) network simulation tool: `go get github.com/tylertreat/comcast` + +Also, to keep track of the data files and logs for your cluster, you may want to create a new directory (e.g., `mkdir follow-workload`) and start all your nodes in that directory. + +## Step 2. Start the cluster + +Use the [`cockroach start`](cockroach-start.html) command to start 3 nodes on your local workstation, using the [`--locality`](cockroach-start.html#locality) flag to pretend that each node is in a different region of the US. + +1. Start a node in the "US West": + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --locality=region=us-west \ + --store=follow1 \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +2. Start a node in the "US Midwest": + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --locality=region=us-midwest \ + --store=follow2 \ + --listen-addr=localhost:26258 \ + --http-addr=localhost:8081 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +3. Start a node in the "US East": + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --locality=region=us-east \ + --store=follow3 \ + --listen-addr=localhost:26259 \ + --http-addr=localhost:8082 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +4. Use the [`cockroach init`](cockroach-init.html) command to perform a one-time initialization of the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init \ + --insecure \ + --host=localhost:26257 + ~~~ + +## Step 3. Simulate network latency + +"Follow-the-workload" only kicks in when there's high latency between the nodes of the CockroachDB cluster. In this tutorial, you'll run 3 nodes on your local workstation, with each node pretending to be in a different region of the US. To simulate latency between the nodes, use the `comcast` tool that you installed earlier. + +1. Start `comcast` as follows: + + {% include copy-clipboard.html %} + ~~~ shell + $ comcast --device lo0 --latency 100 + ~~~ + + For the `--device` flag, use `lo0` if you're on Mac or `lo` if you're on Linux. If neither works, run the `ifconfig` command and find the interface responsible for `127.0.0.1` in the output. + + This command causes a 100 millisecond delay for all requests on the loopback interface of your local workstation. It will only affect connections from the machine to itself, not to/from the Internet. + +2. To verify the delay between nodes, check the **Network Latency** page of the Admin UI: + + CockroachDB Admin UI + +## Step 4. Simulate traffic in the US East + +Now that the cluster is live, use CockroachDB's [built-in version the `tpcc` benchmark](cockroach-workload.html) to simulate multiple client connections to the node in the "US East". + +1. Load the initial schema and data, pointing it at port `26259`, which is the port of the node with the `us-east` locality: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init tpcc \ + 'postgresql://root@localhost:26259?sslmode=disable' + ~~~ + +2. Let the workload run to completion. + +## Step 5. Check the location of the range lease + +The load generator created a `tpcc` database with several tables that map to underlying key-value ranges. Verify that the range lease for the `customer` table moved to the node in the "US East" as follows. + +1. Run the [`cockroach node status`](cockroach-node.html) command against any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach node status --insecure --host=localhost:26259 + ~~~ + + ~~~ + id | address | sql_address | build | started_at | updated_at | locality | is_available | is_live + +----+-----------------+-----------------+-----------------------------------------+----------------------------------+----------------------------------+-------------------+--------------+---------+ + 1 | localhost:26257 | localhost:26257 | v19.2.0-alpha.20190606-2491-gfe735c9a97 | 2019-09-28 03:14:20.566372+00:00 | 2019-09-28 03:18:41.866604+00:00 | region=us-west | true | true + 2 | localhost:26259 | localhost:26259 | v19.2.0-alpha.20190606-2491-gfe735c9a97 | 2019-09-28 03:14:21.353188+00:00 | 2019-09-28 03:18:38.165272+00:00 | region=us-east | true | true + 3 | localhost:26258 | localhost:26258 | v19.2.0-alpha.20190606-2491-gfe735c9a97 | 2019-09-28 03:14:21.862969+00:00 | 2019-09-28 03:18:38.577831+00:00 | region=us-midwest | true | true + (3 rows) + ~~~ + +2. In the response, note the ID of the node running on port `26259` (in this case, node 2). + +3. Connect the [built-in SQL shell](cockroach-sql.html) to any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost:26259 + ~~~ + +4. Check where the range lease is for the `tpcc.customer` table: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW RANGES FROM TABLE tpcc.customer; + ~~~ + + ~~~ + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-----------------------+----------+---------------------------------------------------+ + NULL | NULL | 28 | 19.194141 | 2 | region=us-east | {1,2,3} | {region=us-west,region=us-east,region=us-midwest} + (1 row) + ~~~ + + `lease_holder` and `replicas` indicate the node IDs. As you can see, the lease for the range holding the `customer` table's data is on node 2, which is the same ID as the node on port `26259`. + +5. Exit the SQL shell: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +## Step 6. Simulate traffic in the US West + +1. Run the [`cockroach workload run tpcc`](cockroach-workload.html) command to generate more load, this time pointing it at port `26257`, which is the port of the node with the `us-west` locality: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload run tpcc \ + --duration=5m \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + + You'll see per-operation statistics print to standard output every second. + +2. Let the workload run to completion. This is necessary since the system will still "remember" the earlier requests to the other locality. + + {{site.data.alerts.callout_info}} + The latency numbers printed by the workload will be over 200 milliseconds because the 100 millisecond delay in each direction (200ms round-trip) caused by the `comcast` tool also applies to the traffic going from the `tpcc` process to the `cockroach` process. If you were to set up more advanced rules that excluded the `tpcc` process's traffic or to run this on a real network with real network delay, these numbers would be down in the single-digit milliseconds. + {{site.data.alerts.end}} + +## Step 7. Check the location of the range lease + +Verify that the range lease for the `customer` table moved to the node in the "US West" as follows. + +1. Connect the [built-in SQL shell](cockroach-sql.html) to any node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost:26257 + ~~~ + +2. Check where the range lease is for the `tpcc.customer` table: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW RANGES FROM TABLE tpcc.customer; + ~~~ + + ~~~ + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-----------------------+----------+---------------------------------------------------+ + NULL | NULL | 28 | 19.194141 | 1 | region=us-west | {1,2,3} | {region=us-west,region=us-east,region=us-midwest} + (1 row) + ~~~ + + As you can see, the lease for the range holding the `customer` table's data is now on node 1, which is the same ID as the node on port `26257`. + +## Step 8. Clean up + +1. Once you're done with this tutorial, you will not want a 100 millisecond delay for all requests on your local workstation, so stop the `comcast` tool: + + {% include copy-clipboard.html %} + ~~~ shell + $ comcast --device lo0 --stop + ~~~ + +2. Use the [`cockroach quit`](cockroach-quit.html) command to gracefully shut down each node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26257 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26258 + ~~~ + + {{site.data.alerts.callout_info}} + For the last node, the shutdown process will take longer (about a minute each) and will eventually force the node to stop. This is because, with only 1 of 3 nodes left, a majority of replicas are not available, and so the cluster is no longer operational. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26259 + ~~~ + +3. To restart the cluster at a later time, run the same `cockroach start` commands as earlier from the directory containing the nodes' data stores. + + If you do not plan to restart the cluster, you may want to remove the nodes' data stores: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm -rf follow1 follow2 follow3 + ~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} diff --git a/v20.2/demo-json-support.md b/v20.2/demo-json-support.md new file mode 100644 index 00000000000..166e3eefb01 --- /dev/null +++ b/v20.2/demo-json-support.md @@ -0,0 +1,271 @@ +--- +title: JSON Support +summary: Use a local cluster to explore how CockroachDB can store and query unstructured JSONB data. +toc: true +--- + +This page walks you through a simple demonstration of how CockroachDB can store and query unstructured [`JSONB`](jsonb.html) data from a third-party API, as well as how an [inverted index](inverted-indexes.html) can optimize your queries. + +## Step 1. Install prerequisites + +
+ + +
+ +
+- Install the latest version of [CockroachDB](install-cockroachdb.html). +- Install the latest version of [Go](https://golang.org/dl/): `brew install go` +- Install the [PostgreSQL driver](https://github.com/lib/pq): `go get github.com/lib/pq` +
+ +
+- Install the latest version of [CockroachDB](install-cockroachdb.html). +- Install the [Python psycopg2 driver](http://initd.org/psycopg/docs/install.html): `pip install psycopg2` +- Install the [Python Requests library](http://docs.python-requests.org/en/master/): `pip install requests` +
+ +## Step 2. Start a single-node cluster + +For the purpose of this tutorial, you need only one CockroachDB node running in insecure mode, so use the [`cockroach start-single-node`](cockroach-start-single-node.html) command: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start-single-node \ +--insecure \ +--store=json-test \ +--listen-addr=localhost:26257 \ +--http-addr=localhost:8080 \ +--background +~~~ + +## Step 3. Create a user + +Open the [built-in SQL shell](cockroach-sql.html) as the `root` user and create a new user, `maxroach`: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=localhost:26257 +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE USER maxroach; +~~~ + +## Step 4. Create a database and grant privileges + +Next, create a database called `jsonb_test`: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE DATABASE jsonb_test; +~~~ + +Set the database as the default: + +{% include copy-clipboard.html %} +~~~ sql +> SET DATABASE = jsonb_test; +~~~ + +Then [grant privileges](grant.html) to the `maxroach` user: + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ALL ON DATABASE jsonb_test TO maxroach; +~~~ + +## Step 5. Create a table + +Still in the SQL shell, create a table called `programming`: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE programming ( + id UUID DEFAULT uuid_v4()::UUID PRIMARY KEY, + posts JSONB + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE programming; +~~~ + +~~~ + table_name | create_statement ++-------------+------------------------------------------------+ + programming | CREATE TABLE programming ( + | id UUID NOT NULL DEFAULT uuid_v4()::UUID, + | posts JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (id ASC), + | FAMILY "primary" (id, posts) + | ) +(1 row) +~~~ + +## Step 6. Run the code + +Now that you have a database, user, and a table, let's run code to insert rows into the table. + +
+ + +
+ +
+The code queries the [Reddit API](https://www.reddit.com/dev/api/) for posts in [/r/programming](https://www.reddit.com/r/programming/). The Reddit API only returns 25 results per page; however, each page returns an `"after"` string that tells you how to get the next page. Therefore, the program does the following in a loop: + +1. Makes a request to the API. +2. Inserts the results into the table and grabs the `"after"` string. +3. Uses the new `"after"` string as the basis for the next request. + +Download the json-sample.go file, or create the file yourself and copy the code into it: + +{% include copy-clipboard.html %} +~~~ go +{% include {{ page.version.version }}/json/json-sample.go %} +~~~ + +In a new terminal window, navigate to your sample code file and run it: + +{% include copy-clipboard.html %} +~~~ shell +$ go run json-sample.go +~~~ +
+ +
+The code queries the [Reddit API](https://www.reddit.com/dev/api/) for posts in [/r/programming](https://www.reddit.com/r/programming/). The Reddit API only returns 25 results per page; however, each page returns an `"after"` string that tells you how to get the next page. Therefore, the program does the following in a loop: + +1. Makes a request to the API. +2. Grabs the `"after"` string. +3. Inserts the results into the table. +4. Uses the new `"after"` string as the basis for the next request. + +Download the json-sample.py file, or create the file yourself and copy the code into it: + +{% include copy-clipboard.html %} +~~~ python +{% include {{ page.version.version }}/json/json-sample.py %} +~~~ + +In a new terminal window, navigate to your sample code file and run it: + +{% include copy-clipboard.html %} +~~~ shell +$ python json-sample.py +~~~ +
+ +The program will take awhile to finish, but you can start querying the data right away. + +## Step 7. Query the data + +Back in the terminal where the SQL shell is running, verify that rows of data are being inserted into your table: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT count(*) FROM programming; +~~~ + +~~~ + count ++-------+ + 675 +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT count(*) FROM programming; +~~~ + +~~~ + count ++-------+ + 825 +(1 row) +~~~ + +Now, retrieve all the current entries where the link is pointing to somewhere on GitHub: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id FROM programming \ +WHERE posts @> '{"data": {"domain": "github.com"}}'; +~~~ + +~~~ + id ++--------------------------------------+ + 05348629-d8f1-4c90-99cc-11e8ab313edb + 059a1562-0054-49ff-adc7-aec82c6f74fb + 1b5ea86d-c892-43ba-b40a-c63761aff3ea + 25ac5bfe-44e2-4c6a-892c-959f859ee4e7 + 2ab49796-3e55-4a33-8a83-9decef9fbccc + 2df2e3ac-757b-4689-844d-935876df75e9 + 4506e0b8-a572-499c-a9c1-2a5075a021f8 + 5209ce99-2253-4490-bceb-fd881ff6d962 + 56cf90cd-43a9-49e9-a078-3e28c115232f + 57f287a3-d396-460a-a649-9fa41c4315e4 + ... +(90 rows) + +Time: 103.748ms +~~~ + +{{site.data.alerts.callout_info}} +Since you are querying live data, your results for this and the following steps may vary from the results documented in this tutorial. +{{site.data.alerts.end}} + +## Step 8. Create an inverted index to optimize performance + +The query in the previous step took 103.748ms. To optimize the performance of queries that filter on the `JSONB` column, let's create an [inverted index](inverted-indexes.html) on the column: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INVERTED INDEX ON programming(posts); +~~~ + +## Step 9. Run the query again + +Now that there is an inverted index, the same query will run much faster: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT id FROM programming \ +WHERE posts @> '{"data": {"domain": "github.com"}}'; +~~~ +~~~ +(109 rows) + +Time: 6.862ms +~~~ + +Instead of 103.748ms, the query now takes 6.862ms. + +## Step 10. Clean up + +If the program is still running, press CTRL + c to stop it and then use [`cockroach quit`](cockroach-quit.html) to stop the single-node CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --insecure --host=localhost:26257 +~~~ + +If you do not plan to restart the cluster, remove the node's data store: + +{% include copy-clipboard.html %} +~~~ shell +$ rm -rf json-test +~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} + +You may also want to learn more about the [`JSONB`](jsonb.html) data type and [inverted indexes](inverted-indexes.html). diff --git a/v20.2/demo-low-latency-multi-region-deployment.md b/v20.2/demo-low-latency-multi-region-deployment.md new file mode 100644 index 00000000000..6f876955774 --- /dev/null +++ b/v20.2/demo-low-latency-multi-region-deployment.md @@ -0,0 +1,1156 @@ +--- +title: Low Latency Reads and Writes in a Multi-Region Cluster +summary: Use data topologies to get low-latency reads and writes in a multi-region CockroachDB cluster. +toc: true +toc_not_nested: true +redirect_from: demo-geo-partitioning.html +key: demo-geo-partitioning.html +--- + +In CockroachDB, data is replicated and distributed across the nodes of a cluster for consistency and resiliency, and [read and write requests are automatically routed between nodes](architecture/reads-and-writes-overview.html) as appropriate. In a single-region cluster, this behavior doesn't affect performance because network latency between nodes is sub-millisecond. In a cluster spread across multiple geographic regions, however, the distribution of data becomes a key performance bottleneck, and for that reason, it is important to think about the latency requirements of each table and then use the appropriate [data topologies](topology-patterns.html) to locate data for optimal performance. + +This tutorial walks you through the process of deploying a 9-node CockroachDB cluster across 3 US regions, 3 AZs per region, with a fictional vehicle-sharing application called [MovR](movr.html) running concurrently in each region. Initially, you'll see the effect of network latency when requests must move back and forth across the US. Then you'll use two important multi-region data topologies, [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) and [Duplicate Indexes](topology-duplicate-indexes.html), to remove this bottleneck and dramatically lower latency, with the majority of reads and writes executing in 2 milliseconds or less. Finally, you'll experience the cluster's resiliency to AZ-level failure. + +## See it in action + +### Watch a demo + +Watch [this webinar recording](https://www.cockroachlabs.com/webinars/implementation-topologies-for-distributed-sql +) to see a demonstration of the concepts and features in this tutorial. + + + +### Read a case study + +Read about how an [electronic lock manufacturer](https://www.cockroachlabs.com/case-studies/european-electronic-lock-manufacturer-modernizes-iam-system-with-managed-cockroachdb/) and [multi-national bank](https://www.cockroachlabs.com/case-studies/top-five-multinational-bank-modernizes-its-european-core-banking-services-migrating-from-oracle-to-cockroachdb/) are using the Geo-Partitioned Replicas topology in production for improved performance and regulatory compliance. + +## Before you begin + +- [Request a trial license](#request-a-trial-license) +- [Review important concepts](#review-important-concepts) +- [Review the cluster setup](#review-the-cluster-setup) +- [Review the MovR application](#review-the-movr-application) + +### Request a trial license + +Some CockroachDB features used in this tutorial require an enterprise license, so [request a 30-day trial license](https://www.cockroachlabs.com/get-cockroachdb/) before you get started. + +You should receive your trial license via email within a few minutes. You'll enable your license once your cluster is up-and-running. + +### Review important concepts + +To understand performance in a geographically distributed CockroachDB cluster, it's important to first review [how reads and writes work in CockroachDB](architecture/reads-and-writes-overview.html). + +### Review the cluster setup + +You'll deploy a 9-node CockroachDB cluster across 3 GCE regions, with each node on a VM in a distinct availability zone for optimal resiliency: + +Multi-region hardware setup + +A few notes: + +- For each CockroachDB node, you'll use the [`n1-standard-4`](https://cloud.google.com/compute/docs/machine-types#standard_machine_types) machine type (4 vCPUs, 15 GB memory) with the Ubuntu 16.04 OS image and a [local SSD](https://cloud.google.com/compute/docs/disks/local-ssd#create_local_ssd) disk. +- You'll start each node with the [`--locality` flag](cockroach-start.html#locality) describing the node's region and availability zone. Initially, this locality information will lead CockroachDB to evenly distribute data across the 3 regions. Then, it will be used to apply data topologies for lower latency. +- There will be an extra VM in each region for an instance of the MovR application and the open-source HAProxy load balancer. The application in each region will be pointed at the local load balancer, which will direct connections only to the CockroachDB nodes in the same region. + +### Review the MovR application + +For your application, you'll use our open-source, fictional, peer-to-peer vehicle-sharing app, [MovR](movr.html). You'll run 3 instances of MovR, one in each US region, with each instance representing users in a specific city: New York, Chicago, or Seattle. + +#### The schema + +{% include {{ page.version.version }}/misc/movr-schema.md %} + +All of the tables except `promo_codes` have a multi-column primary key of `city` and `id`, with `city` being the first in the key. As such, the rows in these tables are geographically specific and ordered by geography. These tables are read and updated very frequently, and so to keep read and write latency low, you'll use the [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) topology for these tables. + +In contrast, the data in the `promo_codes` table is not tied to geography, and the data is read frequently but rarely updated. This type of table is often referred to as a "reference table" or "lookup table". In this case, you'll use the [Duplicate Indexes](topology-duplicate-indexes.html) topology to keep just read latency very low, since that's primary. + +#### The workflow + +{% include {{ page.version.version }}/misc/movr-workflow.md %} + +## Step 1. Set up the environment + +- [Configure your network](#configure-your-network) +- [Provision VMs](#provision-vms) + +### Configure your network + +{% include {{ page.version.version }}/performance/configure-network.md %} + +### Provision VMs + +You need 9 VMs across 3 GCE regions, 3 per region with each VM in a distinct availability zone. You also need 3 extra VMs, 1 per region, for a region-specific version of MovR and the HAProxy load balancer. + +1. [Create 9 VMs](https://cloud.google.com/compute/docs/instances/create-start-instance) for CockroachDB nodes. + + When creating each VM: + - Use the [`n1-standard-4`](https://cloud.google.com/compute/docs/machine-types#standard_machine_types) machine type (4 vCPUs, 15 GB memory) and the Ubuntu 16.04 OS image. + - Select one of the following [region and availability zone](https://cloud.google.com/compute/docs/regions-zones/) configurations. Be sure to use each region/availability combination only once. + + VM | Region | Availability Zone + ---|--------|------------------ + 1 | `us-east1` | `us-east1-b` + 2 | `us-east1` | `us-east1-c` + 3 | `us-east1` | `us-east1-d` + 4 | `us-central1` | `us-central1-a` + 5 | `us-central1` | `us-central1-b` + 6 | `us-central1` | `us-central1-c` + 7 | `us-west1` | `us-west1-a` + 8 | `us-west1` | `us-west1-b` + 9 | `us-west1` | `us-west1-c` + - [Create and mount a local SSD](https://cloud.google.com/compute/docs/disks/local-ssd#create_local_ssd). + - To apply the Admin UI firewall rule you created earlier, click **Management, disk, networking, SSH keys**, select the **Networking** tab, and then enter `cockroachdb` in the **Network tags** field. + +2. [Create 3 VMs](https://cloud.google.com/compute/docs/instances/create-start-instance) for the region-specific versions of MovR and HAProxy, one in each of the regions mentioned above, using same machine types and OS image as mentioned above. + +3. Note the internal IP address of each VM. You'll need these addresses when starting the CockroachDB nodes, configuring HAProxy, and running the MovR application. + +## Step 2. Start CockroachDB + +Now that you have VMs in place, start your CockroachDB cluster across the three US regions. + +- [Start nodes in US East](#start-nodes-in-us-east) +- [Start nodes in US Central](#start-nodes-in-us-central) +- [Start nodes in US West](#start-nodes-in-us-west) +- [Initialize the cluster](#initialize-the-cluster) + +### Start nodes in US East + +1. SSH to the first VM in the US East region where you want to run a CockroachDB node. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +3. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --advertise-addr= \ + --join=:26257,:26257,:26257 \ + --locality=cloud=gce,region=us-east1,zone= \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +4. Repeat steps 1 - 3 for the other two CockroachDB nodes in the region. Each time, be sure to: + - Adjust the `--advertise-addr` flag. + - Use the appropriate availability zone of the VM in the `zone` portion of the `--locality` flag. + +### Start nodes in US Central + +1. SSH to the first VM in the US Central region where you want to run a CockroachDB node. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +3. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --advertise-addr= \ + --join=:26257,:26257,:26257 \ + --locality=cloud=gce,region=us-central1,zone= \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +4. Repeat steps 1 - 3 for the other two CockroachDB nodes in the region. Each time, be sure to: + - Adjust the `--advertise-addr` flag. + - Use the appropriate availability zone of the VM in the `zone` portion of the `--locality` flag. + +### Start nodes in US West + +1. SSH to the first VM in the US West region where you want to run a CockroachDB node. + +2. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + +3. Run the [`cockroach start`](cockroach-start.html) command: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --advertise-addr= \ + --join=:26257,:26257,:26257 \ + --locality=cloud=gce,region=us-west1,zone= \ + --cache=.25 \ + --max-sql-memory=.25 \ + --background + ~~~ + +4. Repeat steps 1 - 3 for the other two CockroachDB nodes in the region. Each time, be sure to: + - Adjust the `--advertise-addr` flag. + - Use the appropriate availability zone of the VM in the `zone` portion of the `--locality` flag. + +### Initialize the cluster + +On any of the VMs, run the one-time [`cockroach init`](cockroach-init.html) command to join the first nodes into a cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --insecure --host=
+~~~ + +## Step 3. Start MovR + +- [Set up the client VMs](#set-up-the-client-vms) +- [Configure the cluster for MovR](#configure-the-cluster-for-movr) +- [Start MovR in US East](#start-movr-in-us-east) +- [Start MovR in US Central](#start-movr-in-us-central) +- [Start MovR in US West](#start-movr-in-us-west) + +### Set up the client VMs + +Next, install Docker and HAProxy on each client VM. Docker is required so you can later run MovR from a Docker image, and HAProxy will serve as the region-specific load balancer for MovR in each region. + +1. SSH to the VM in the US East region where you want to run MovR and HAProxy. + +2. Install Docker: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get update && \ + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg-agent software-properties-common && \ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - && \ + sudo apt-key fingerprint 0EBFCD88 && \ + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" && \ + sudo apt-get update && \ + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + ~~~ + + If you get an error, run one command at a time or follow the [official Docker instructions](https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-using-the-repository). + +3. Install HAProxy: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get update + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo apt-get install haproxy + ~~~ + +4. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, extract the binary, and copy it into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + The `cockroach` binary needs to be on these VMs so you can run some client commands built into the binary, such as the command in the next step and the command for starting the built-in SQL shell. + +5. Run the [`cockroach gen haproxy`](cockroach-gen.html) command to generate an HAProxy config file, specifying the address of any CockroachDB node and the `--locality` of nodes in the US East region: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach gen haproxy \ + --insecure \ + --host=
\ + --locality=region=us-east1 + ~~~ + + The generated configuration file is called `haproxy.cfg` and looks as follows, with the `server` addresses pre-populated with just the nodes in US East based on the `--locality` flag used: + + ~~~ + global + maxconn 4096 + + defaults + mode tcp + # Timeout values should be configured for your specific use. + # See: https://cbonte.github.io/haproxy-dconv/1.8/configuration.html#4-timeout%20connect + timeout connect 10s + timeout client 1m + timeout server 1m + # TCP keep-alive on client side. Server already enables them. + option clitcpka + + listen psql + bind :26257 + mode tcp + balance roundrobin + option httpchk GET /health?ready=1 + server cockroach1 :26257 check port 8080 + server cockroach2 :26257 check port 8080 + server cockroach3 :26257 check port 8080 + ~~~ + +6. Start HAProxy, with the `-f` flag pointing to the `haproxy.cfg` file: + + {% include copy-clipboard.html %} + ~~~ shell + $ haproxy -f haproxy.cfg & + ~~~ + +7. Repeat the steps above for the client VMs in the other two regions. For each region, be sure to adjust the `--locality` flag when running the `cockroach gen haproxy` command. + +### Configure the cluster for MovR + +Before you can run MovR against the cluster, you must create a `movr` database and enable an enterprise license. + +1. SSH to the client VM in the US East region. + +2. Use the [`cockroach sql`](cockroach-sql.html) command to start the built-in SQL shell, specifying the address of the HAProxy load balancer in the region: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=
+ ~~~ + +3. In the SQL shell, create the `movr` database: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE movr; + ~~~ + +4. Enable the trial license you requested earlier: + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING cluster.organization = ''; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SET CLUSTER SETTING enterprise.license = ''; + ~~~ + +5. Set the longitude and latitude of the regions where you are running CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT into system.locations VALUES + ('region', 'us-east1', 33.836082, -81.163727), + ('region', 'us-central1', 42.032974, -93.581543), + ('region', 'us-west1', 43.804133, -120.554201); + ~~~ + + Inserting these coordinates enables you to visualize your cluster on the [**Node Map**](enable-node-map.html) feature of the Admin UI. + +6. Exit the SQL shell: + + {% include copy-clipboard.html %} + ~~~ sql + \q + ~~~ + +### Start MovR in US East + +{{site.data.alerts.callout_info}} +Be sure to use the exact version of MovR specified in the commands: `movr:19.09.2`. This tutorial relies on the SQL schema in this specific version. +{{site.data.alerts.end}} + +1. Still on the client VM in the US East region, load the MovR schema and initial data for the cities of New York, Chicago, and Seattle, pointing at the address of the US East load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo docker run -it --rm cockroachdb/movr:19.09.2 \ + --app-name "movr-load" \ + --url "postgres://root@
:26257/movr?sslmode=disable" \ + load \ + --num-users 100 \ + --num-rides 100 \ + --num-vehicles 10 \ + --city-pair us_east:"new york" \ + --city-pair central:chicago \ + --city-pair us_west:seattle + ~~~ + + After the Docker image downloads, you'll see data being generated for the specified cities: + + ~~~ + ... + [INFO] (MainThread) initializing tables + [INFO] (MainThread) loading cities ['new york', 'chicago', 'seattle'] + [INFO] (MainThread) loading movr data with ~100 users, ~10 vehicles, and ~100 rides + [INFO] (MainThread) Only using 3 of 5 requested threads, since we only create at most one thread per city + [INFO] (Thread-1 ) Generating user data for new york... + [INFO] (Thread-2 ) Generating user data for chicago... + [INFO] (Thread-3 ) Generating user data for seattle... + [INFO] (Thread-2 ) Generating vehicle data for chicago... + [INFO] (Thread-3 ) Generating vehicle data for seattle... + [INFO] (Thread-1 ) Generating vehicle data for new york... + [INFO] (Thread-2 ) Generating ride data for chicago... + [INFO] (Thread-3 ) Generating ride data for seattle... + [INFO] (Thread-1 ) Generating ride data for new york... + [INFO] (Thread-2 ) populated chicago in 9.173931 seconds + [INFO] (Thread-3 ) populated seattle in 9.257723 seconds + [INFO] (Thread-1 ) populated new york in 9.386243 seconds + [INFO] (MainThread) populated 3 cities in 20.587325 seconds + [INFO] (MainThread) - 4.954505 users/second + [INFO] (MainThread) - 4.954505 rides/second + [INFO] (MainThread) - 0.582883 vehicles/second + ~~~ + +2. Start MovR in the US East region, representing users in New York. Be sure to point at the address of the US East load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo docker run -it --rm cockroachdb/movr:19.09.2 \ + --app-name "movr-east" \ + --url "postgres://root@
:26257/movr?sslmode=disable" \ + --num-threads=15 \ + run \ + --city="new york" + ~~~ + +### Start MovR in US Central + +1. SSH to the client VM in the US Central region. + +2. Start MovR in the US Central region, representing users in Chicago. Be sure to point at the address of the US Central load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo docker run -it --rm cockroachdb/movr:19.09.2 \ + --app-name "movr-central" \ + --url "postgres://root@
:26257/movr?sslmode=disable" \ + --num-threads=15 \ + run \ + --city="chicago" + ~~~ + +### Start MovR in US West + +1. SSH to the client VM in the US West region. + +2. Start MovR in the US West region, representing users in Seattle. Be sure to point at the address of the US West load balancer: + + {% include copy-clipboard.html %} + ~~~ shell + $ sudo docker run -it --rm cockroachdb/movr:19.09.2 \ + --app-name "movr-west" \ + --url "postgres://root@
:26257/movr?sslmode=disable" \ + --num-threads=15 \ + run \ + --city="seattle" + ~~~ + +## Step 4. Access the Admin UI + +Now that you've deployed and configured your cluster, take a look at it in the Admin UI: + +1. Open a browser and go to `http://:8080`. + +2. On the **Cluster Overview** page, select **View: Node Map** to access the [Node Map](enable-node-map.html), which visualizes your CockroachDB cluster on a map of the US: + + Geo-partitioning node map + +3. Drill down one level to see your nodes across 3 regions: + + Geo-partitioning node map + +4. Drill into a region to see that each node is in a distinct availability zone: + + Geo-partitioning node map + +## Step 5. Check latency + +Use the Admin UI to see the effect of network latency before applying multi-region data topologies. + +1. Still in the Admin UI, click **Metrics** on the left and hover over the **Service Latency: SQL, 99th percentile** timeseries graph: + + Geo-partitioning SQL latency + + For each node, you'll see that the max latency of 99% of queries is in the 100s of milliseconds. To understand why SQL latency is so high, it's important to first look at how long it takes requests to physically travel between the nodes in your cluster. + +2. Click **Network Latency** in the left-hand navigation: + + Geo-partitioning network latency + + The **Network Latency** page shows the round-trip latency between any two nodes in your cluster. Here's a node/region mapping: + + Nodes | Region + ------|------- + 1 - 3 | `us-east1` + 4 - 6 | `us-central1` + 7 - 9 | `us-west1` + + As you can see, within a single region, round-trip latency is sub-millisecond. For example, between nodes 5 and 6 in the `us-central1` region, round-trip latency is 0.56ms. However, between nodes in different regions, round-trip latency is significantly higher. For example, between node 2 in `us-east1` and node 7 in `us-west`, round-trip latency is 66.43ms. + +## Step 6. Check replica distribution + +With network latency in mind, now use the built-in SQL shell to check the distribution of replicas. This will help us understand how SQL queries are moving between the nodes of the cluster and, thus, incurring latency. + +1. SSH to the client VM in any region. + +2. Use the [`cockroach sql`](cockroach-sql.html) command to start the built-in SQL shell, specifying the address of the HAProxy load balancer in the region: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --database=movr --host=
+ ~~~ + +3. In the SQL shell, use the [`SHOW RANGES`](show-ranges.html) statement to view the location of replicas for the tables and their secondary indexes: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW RANGES FROM TABLE users; + SHOW RANGES FROM TABLE vehicles; + SHOW RANGES FROM INDEX vehicles_auto_index_fk_city_ref_users; + SHOW RANGES FROM TABLE rides; + SHOW RANGES FROM INDEX rides_auto_index_fk_city_ref_users; + SHOW RANGES FROM INDEX rides_auto_index_fk_vehicle_city_ref_vehicles; + SHOW RANGES FROM TABLE user_promo_codes; + SHOW RANGES FROM TABLE vehicle_location_histories; + ~~~ + + ~~~ + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 85 | 0.158604 | 1 | cloud=gce,region=us-east1,zone=us-east1-b | {1,6,8} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-b"} + (1 row) + + Time: 750.045316ms + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 37 | 0.298143 | 3 | cloud=gce,region=us-east1,zone=us-east1-d | {3,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-d","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 473.718371ms + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 37 | 0.302683 | 3 | cloud=gce,region=us-east1,zone=us-east1-d | {3,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-d","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 2.556900719s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 39 | 0.563349 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {3,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-d","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 673.337559ms + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 39 | 0.571556 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {3,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-d","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 3.184113514s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 39 | 0.57792 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {3,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-d","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 2.812128768s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 38 | 0.054887 | 9 | cloud=gce,region=us-west1,zone=us-west1-c | {2,6,9} | {"cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-c"} + (1 row) + + Time: 896.010317ms + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 86 | 2.197216 | 6 | cloud=gce,region=us-central1,zone=us-central1-c | {2,6,7} | {"cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-a"} + (1 row) + + Time: 708.643369ms + ~~~ + + Here's the node/region mapping again: + + Nodes | Region + ------|------- + 1 - 3 | `us-east1` + 4 - 6 | `us-central1` + 7 - 9 | `us-west1` + + You'll see that most tables and indexes map to a single range, and each range has 3 replicas spread across regions with one replica identified as `lease_holder`. + + Thinking back to [how reads and writes work in CockroachDB](architecture/reads-and-writes-overview.html), this tells you that many reads are leaving their region to reach the relevant leaseholder replica, and all writes are spanning regions to achieve Raft consensus. This explains the currently high latencies. + + For example, based on the output above, the replicas for the `users` table are on nodes 1, 6, and 8, with the leaseholder on node 1. This means that when a user in Seattle registers for the MovR service: + + 1. A request to write a row to the `users` table goes through the load balancer in US west to a gateway node in US west. + 2. The request is routed to the leaseholder on node 1 in US east. + 3. The leaseholder waits for consensus from a replica in US central or US west. + 4. The leaseholder returns acknowledgement to the gateway node in the US west. + 5. The gateway node responds to the client. + +## Step 7. Apply data topologies + +- [Partition geo-specific tables](#partition-geo-specific-tables) +- [Duplicate the reference table](#duplicate-the-reference-table) + +### Partition geo-specific tables + +As mentioned earlier, all of the tables except `promo_codes` are geographically specific, ordered by `city`, and read and updated very frequently. For these tables, the most effective way to prevent the high latency resulting from cross-region operations is to apply the [Geo-Partitioned Replicas](topology-geo-partitioned-replicas.html) data topology. In practice, you will tell CockroachDB to partition these tables and their secondary indexes by `city`, each partition becoming its own range of 3 replicas. You will then tell CockroachDB to pin each partition (all of its replicas) to the relevant region. This means that reads and writes on these tables will always have access to the relevant replicas in each region and, therefore, will have low, intra-region latencies. + +1. Back in the SQL shell on one of your client VMs, use [`ALTER TABLE/INDEX ... PARTITION BY`](partition-by.html) statements to define partitions by `city` for the geo-specific tables and their secondary indexes: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE users PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE vehicles PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER INDEX vehicles_auto_index_fk_city_ref_users PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE rides PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER INDEX rides_auto_index_fk_city_ref_users PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER INDEX rides_auto_index_fk_vehicle_city_ref_vehicles PARTITION BY LIST (vehicle_city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE user_promo_codes PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE vehicle_location_histories PARTITION BY LIST (city) ( + PARTITION new_york VALUES IN ('new york'), + PARTITION chicago VALUES IN ('chicago'), + PARTITION seattle VALUES IN ('seattle') + ); + ~~~ + +2. Use the [`SHOW CREATE TABLE`](show-create.html) statement to review the partition definition for one of the geo-specific tables: + + {{site.data.alerts.callout_success}} + The warning at the bottom tells you that partitions are not yet applied because corresponding replication zones still need to be created. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW CREATE TABLE vehicles; + ~~~ + + ~~~ + table_name | create_statement + +------------+-----------------------------------------------------------------------------------------------------+ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | type VARCHAR NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status VARCHAR NULL, + | current_location VARCHAR NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | CONSTRAINT fk_city_ref_users FOREIGN KEY (city, owner_id) REFERENCES users(city, id), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC) PARTITION BY LIST (city) ( + | PARTITION new_york VALUES IN (('new york')), + | PARTITION chicago VALUES IN (('chicago')), + | PARTITION seattle VALUES IN (('seattle')) + | ), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) PARTITION BY LIST (city) ( + | PARTITION new_york VALUES IN (('new york')), + | PARTITION chicago VALUES IN (('chicago')), + | PARTITION seattle VALUES IN (('seattle')) + | ) + | -- Warning: Partitioned table with no zone configurations. + (1 row) + ~~~ + +3. Use [`ALTER PARTITION ... CONFIGURE ZONE`](configure-zone.html) statements to create replication zones that pin each partition to nodes in the relevant region, using the localities specified when nodes were started: + + {{site.data.alerts.callout_success}} + The `
@*` syntax lets you create zone configurations for all identically named partitions of a table, saving you multiple steps. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER PARTITION new_york OF INDEX movr.users@* + CONFIGURE ZONE USING constraints='[+region=us-east1]'; + ALTER PARTITION chicago OF INDEX movr.users@* + CONFIGURE ZONE USING constraints='[+region=us-central1]'; + ALTER PARTITION seattle OF INDEX movr.users@* + CONFIGURE ZONE USING constraints='[+region=us-west1]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER PARTITION new_york OF INDEX movr.vehicles@* + CONFIGURE ZONE USING constraints='[+region=us-east1]'; + ALTER PARTITION chicago OF INDEX movr.vehicles@* + CONFIGURE ZONE USING constraints='[+region=us-central1]'; + ALTER PARTITION seattle OF INDEX movr.vehicles@* + CONFIGURE ZONE USING constraints='[+region=us-west1]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER PARTITION new_york OF INDEX movr.rides@* + CONFIGURE ZONE USING constraints='[+region=us-east1]'; + ALTER PARTITION chicago OF INDEX movr.rides@* + CONFIGURE ZONE USING constraints='[+region=us-central1]'; + ALTER PARTITION seattle OF INDEX movr.rides@* + CONFIGURE ZONE USING constraints='[+region=us-west1]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER PARTITION new_york OF INDEX movr.user_promo_codes@* + CONFIGURE ZONE USING constraints='[+region=us-east1]'; + ALTER PARTITION chicago OF INDEX movr.user_promo_codes@* + CONFIGURE ZONE USING constraints='[+region=us-central1]'; + ALTER PARTITION seattle OF INDEX movr.user_promo_codes@* + CONFIGURE ZONE USING constraints='[+region=us-west1]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER PARTITION new_york OF INDEX movr.vehicle_location_histories@* + CONFIGURE ZONE USING constraints='[+region=us-east1]'; + ALTER PARTITION chicago OF INDEX movr.vehicle_location_histories@* + CONFIGURE ZONE USING constraints='[+region=us-central1]'; + ALTER PARTITION seattle OF INDEX movr.vehicle_location_histories@* + CONFIGURE ZONE USING constraints='[+region=us-west1]'; + ~~~ + +3. At this point, you can use the [`SHOW CREATE TABLE`](show-create.html) statement to confirm that partitions are in effect: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW CREATE TABLE vehicles; + ~~~ + + ~~~ + table_name | create_statement + +------------+-------------------------------------------------------------------------------------------------------------------+ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | type VARCHAR NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status VARCHAR NULL, + | current_location VARCHAR NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | CONSTRAINT fk_city_ref_users FOREIGN KEY (city, owner_id) REFERENCES users(city, id), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC) PARTITION BY LIST (city) ( + | PARTITION new_york VALUES IN (('new york')), + | PARTITION chicago VALUES IN (('chicago')), + | PARTITION seattle VALUES IN (('seattle')) + | ), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) PARTITION BY LIST (city) ( + | PARTITION new_york VALUES IN (('new york')), + | PARTITION chicago VALUES IN (('chicago')), + | PARTITION seattle VALUES IN (('seattle')) + | ); + | ALTER PARTITION chicago OF INDEX movr.public.vehicles@primary CONFIGURE ZONE USING + | constraints = '[+region=us-central1]'; + | ALTER PARTITION new_york OF INDEX movr.public.vehicles@primary CONFIGURE ZONE USING + | constraints = '[+region=us-east1]'; + | ALTER PARTITION seattle OF INDEX movr.public.vehicles@primary CONFIGURE ZONE USING + | constraints = '[+region=us-west1]'; + | ALTER PARTITION chicago OF INDEX movr.public.vehicles@vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING + | constraints = '[+region=us-central1]'; + | ALTER PARTITION new_york OF INDEX movr.public.vehicles@vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING + | constraints = '[+region=us-east1]'; + | ALTER PARTITION seattle OF INDEX movr.public.vehicles@vehicles_auto_index_fk_city_ref_users CONFIGURE ZONE USING + | constraints = '[+region=us-west1]' + (1 row) + ~~~ + + In contrast to the last time you ran this statement, you can now see the commands for re-creating the replication zone for each partition of the `vehicles` table and its secondary index. + + The [`SHOW PARTITIONS`](show-partitions.html) statement is another way to confirm that partitions are in effect: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW PARTITIONS FROM TABLE vehicles; + ~~~ + + ~~~ + database_name | table_name | partition_name | parent_partition | column_names | index_name | partition_value | zone_config | full_zone_config + +---------------+------------+----------------+------------------+--------------+------------------------------------------------+-----------------+---------------------------------------+----------------------------------------+ + movr | vehicles | new_york | NULL | city | vehicles@primary | ('new york') | constraints = '[+region=us-east1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-east1]', + | | | | | | | | lease_preferences = '[]' + movr | vehicles | new_york | NULL | city | vehicles@vehicles_auto_index_fk_city_ref_users | ('new york') | constraints = '[+region=us-east1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-east1]', + | | | | | | | | lease_preferences = '[]' + movr | vehicles | chicago | NULL | city | vehicles@primary | ('chicago') | constraints = '[+region=us-central1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-central1]', + | | | | | | | | lease_preferences = '[]' + movr | vehicles | chicago | NULL | city | vehicles@vehicles_auto_index_fk_city_ref_users | ('chicago') | constraints = '[+region=us-central1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-central1]', + | | | | | | | | lease_preferences = '[]' + movr | vehicles | seattle | NULL | city | vehicles@primary | ('seattle') | constraints = '[+region=us-west1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-west1]', + | | | | | | | | lease_preferences = '[]' + movr | vehicles | seattle | NULL | city | vehicles@vehicles_auto_index_fk_city_ref_users | ('seattle') | constraints = '[+region=us-west1]' | range_min_bytes = 16777216, + | | | | | | | | range_max_bytes = 67108864, + | | | | | | | | gc.ttlseconds = 90000, + | | | | | | | | num_replicas = 3, + | | | | | | | | constraints = '[+region=us-west1]', + | | | | | | | | lease_preferences = '[]' + (6 rows) + ~~~ + + {% include {{page.version.version}}/sql/crdb-internal-partitions.md %} + +### Duplicate the reference table + +In contrast to the other tables, the `promo_codes` table is not tied to geography, and its data is read frequently but rarely updated. This type of table is often referred to as a "reference table" or "lookup table". For this table, you'll keep read latency low by applying the [Duplicate Indexes](topology-duplicate-indexes.html) data topology. In practice, you will put the leaseholder for the table itself (also called the primary index) in one region, create two secondary indexes on the table, and tell CockroachDB to put the leaseholder for each secondary index in one of the other regions. CockroachDB's [cost-based optimizer](cost-based-optimizer.html) will then make sure that reads from `promo_codes` access the local leaseholder (either for the table itself or for one of the secondary indexes). Writes, however, will still leave the region to get consensus for the table and its secondary indexes, but writes are so rare that this won't impact overall performance. + +1. Create two indexes on the `promo_codes` table, and make them complete copies of the primary index: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE INDEX promo_codes_idx_east ON promo_codes (code) + STORING (description, creation_time, expiration_time, rules); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE INDEX promo_codes_idx_west ON promo_codes (code) + STORING (description, creation_time, expiration_time, rules); + ~~~ + +2. Use [`ALTER TABLE/INDEX ... CONFIGURE ZONE`](configure-zone.html) statements to create replication zones for the primary index and each secondary index, in each case setting a leaseholder preference telling CockroachDB to put the leaseholder for the index in a distinct region: + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER TABLE promo_codes + CONFIGURE ZONE USING + num_replicas = 3, + constraints = '{"+region=us-central1": 1}', + lease_preferences = '[[+region=us-central1]]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER INDEX promo_codes@promo_codes_idx_east + CONFIGURE ZONE USING + num_replicas = 3, + constraints = '{"+region=us-east1": 1}', + lease_preferences = '[[+region=us-east1]]'; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > ALTER INDEX promo_codes@promo_codes_idx_west + CONFIGURE ZONE USING + num_replicas = 3, + constraints = '{"+region=us-west1": 1}', + lease_preferences = '[[+region=us-west1]]'; + ~~~ + +## Step 8. Re-check replica distribution + +1. Still in the SQL shell on one of your client VMs, use the [`SHOW RANGES`](show-ranges.html) statement to check replica placement of the geo-specific tables after partitioning: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM [SHOW RANGES FROM TABLE users] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM TABLE vehicles] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM INDEX vehicles_auto_index_fk_city_ref_users] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM TABLE rides] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM INDEX rides_auto_index_fk_city_ref_users] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM INDEX rides_auto_index_fk_vehicle_city_ref_vehicles] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM TABLE user_promo_codes] + WHERE "start_key" NOT LIKE '%Prefix%'; + SELECT * FROM [SHOW RANGES FROM TABLE vehicle_location_histories] + WHERE "start_key" NOT LIKE '%Prefix%'; + ~~~ + + ~~~ + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 105 | 0.933453 | 3 | cloud=gce,region=us-east1,zone=us-east1-d | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 107 | 0.860034 | 6 | cloud=gce,region=us-central1,zone=us-central1-c | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 109 | 0.895921 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 1.645458616s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 125 | 2.11175 | 2 | cloud=gce,region=us-east1,zone=us-east1-c | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 129 | 1.9099 | 5 | cloud=gce,region=us-central1,zone=us-central1-b | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 56 | 2.04172 | 9 | cloud=gce,region=us-west1,zone=us-west1-c | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 1.260863914s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 127 | 0.119543 | 1 | cloud=gce,region=us-east1,zone=us-east1-b | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 130 | 0.106442 | 5 | cloud=gce,region=us-central1,zone=us-central1-b | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 46 | 0.110188 | 9 | cloud=gce,region=us-west1,zone=us-west1-c | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 3.392228893s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 76 | 2.498621 | 2 | cloud=gce,region=us-east1,zone=us-east1-c | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 83 | 2.243434 | 5 | cloud=gce,region=us-central1,zone=us-central1-b | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 148 | 2.39411 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 1.294584902s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 78 | 0.533722 | 1 | cloud=gce,region=us-east1,zone=us-east1-b | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 82 | 0.477912 | 4 | cloud=gce,region=us-central1,zone=us-central1-a | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 149 | 0.505345 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 3.346661477s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 80 | 0.61871 | 2 | cloud=gce,region=us-east1,zone=us-east1-c | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 84 | 0.547892 | 6 | cloud=gce,region=us-central1,zone=us-central1-c | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 150 | 0.579083 | 7 | cloud=gce,region=us-west1,zone=us-west1-a | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 3.341758512s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 133 | 0.365143 | 1 | cloud=gce,region=us-east1,zone=us-east1-b | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 135 | 0.313355 | 6 | cloud=gce,region=us-central1,zone=us-central1-c | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 137 | 0.343468 | 9 | cloud=gce,region=us-west1,zone=us-west1-c | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 1.105110359s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-------------+-----------------------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------------------+ + /"new york" | /"new york"/PrefixEnd | 112 | 20.852122 | 3 | cloud=gce,region=us-east1,zone=us-east1-d | {1,2,3} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-east1,zone=us-east1-c","cloud=gce,region=us-east1,zone=us-east1-d"} + /"chicago" | /"chicago"/PrefixEnd | 114 | 17.631255 | 4 | cloud=gce,region=us-central1,zone=us-central1-a | {4,5,6} | {"cloud=gce,region=us-central1,zone=us-central1-a","cloud=gce,region=us-central1,zone=us-central1-b","cloud=gce,region=us-central1,zone=us-central1-c"} + /"seattle" | /"seattle"/PrefixEnd | 116 | 19.677135 | 8 | cloud=gce,region=us-west1,zone=us-west1-b | {7,8,9} | {"cloud=gce,region=us-west1,zone=us-west1-a","cloud=gce,region=us-west1,zone=us-west1-b","cloud=gce,region=us-west1,zone=us-west1-c"} + (3 rows) + + Time: 1.612425537s + ~~~ + + You'll see that the replicas for each partition are now located on nodes in the relevant region: + - New York partitions are on nodes 1 - 3 + - Chicago partitions are on nodes 4 - 6 + - Seattle partitions are on nodes 7 - 9 + + This means that requests from users in a city no longer leave the region, thus removing all cross-region latencies. + +2. Now use the [`SHOW RANGES`](show-ranges.html) statement to check replica placement of the `promo_codes` reference table and indexes: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW RANGES FROM TABLE promo_codes; + SHOW RANGES FROM INDEX promo_codes_idx_east; + SHOW RANGES FROM INDEX promo_codes_idx_west; + ~~~ + + ~~~ + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 87 | 0.835044 | 6 | cloud=gce,region=us-central1,zone=us-central1-c | {1,6,9} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-c"} + (1 row) + + Time: 517.443988ms + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 89 | 0.83622 | 1 | cloud=gce,region=us-east1,zone=us-east1-b | {1,6,9} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-c"} + (1 row) + + Time: 2.449771429s + + start_key | end_key | range_id | range_size_mb | lease_holder | lease_holder_locality | replicas | replica_localities + +-----------+---------+----------+---------------+--------------+-------------------------------------------+----------+---------------------------------------------------------------------------------------------------------------------------------------------+ + NULL | NULL | 90 | 0.836372 | 9 | cloud=gce,region=us-west1,zone=us-west1-c | {1,6,9} | {"cloud=gce,region=us-east1,zone=us-east1-b","cloud=gce,region=us-central1,zone=us-central1-c","cloud=gce,region=us-west1,zone=us-west1-c"} + (1 row) + + Time: 2.621930607s + ~~~ + + You'll see that the replicas for each index are spread across regions, with the leaseholders each in a distinct region: + - The leaseholder for the `promo_codes` primary index is on node 6 in US Central + - The leaseholder for the `promo_codes_idx_east` secondary index is on node 1 in US East + - The leaseholder for the `promo_codes_idx_west` secondary index is on node 9 in US West + + As you'll see in a just a bit, with one leaseholder in each region, CockroachDB's cost-based optimizer will make sure that reads always access the local leaseholder, keeping reads from this table very fast. + +## Step 9. Re-check latency + +1. Now that you've verified that replicas are located properly, go back to the Admin UI, click **Metrics** on the left, and hover over the **Service Latency: SQL, 99th percentile** timeseries graph: + + Geo-partitioning SQL latency + + For each node, you'll see that **99% of all queries are now under 4 milliseconds**. + +2. 99th percentile latency can be influenced by occasional slow queries. For a more accurate sense of typical SQL latency, go to the following URL to view a custom graph for 90th percentile latency: + + ~~~ + http://:8080/#/debug/chart?charts=%5B%7B%22metrics%22%3A%5B%7B%22downsampler%22%3A3%2C%22aggregator%22%3A3%2C%22derivative%22%3A0%2C%22perNode%22%3Atrue%2C%22source%22%3A%22%22%2C%22metric%22%3A%22cr.node.sql.exec.latency-p90%22%7D%5D%2C%22axisUnits%22%3A2%7D%5D + ~~~ + + Geo-partitioning SQL latency + + As you can see, **90% of all SQL queries execute in less than 2 milliseconds**. In some cases, latency is even sub-millisecond. + +3. Most of the latency reduction is due to the geo-partitioned tables. However, the duplicate indexes approach for the `promo_codes` table is also relevant. To validate that the cost-based optimizer is picking the appropriate leaseholder from reads from `promo_codes` in each region, click **Statements** on the left, select **APP > MOVR-EAST**, and then click the `SELECT FROM promo_codes` statement: + + Geo-partitioning SQL latency + + In the "Logical Plan" area, note the `table = promo_codes@promo_codes_idx_east` scan. This proves that the cost-based optimizer used the leaseholder for that index and, thus, didn't leave the region for the instance of MovR running in US East. + + To validate this behavior in the other regions, click **Statements** again on the left and follow the same steps for the other apps instances. + +## Step 10. Test resiliency + +There are various resiliency levels in your cluster: + +- For the [geo-partitioned data](topology-geo-partitioned-replicas.html#resiliency), each partition is constrained to a specific region and balanced across the 3 AZs in the region, so one AZ can fail per region without interrupting access to the partitions in that region. +- For the [duplicated reference data](topology-duplicate-indexes.html#resiliency), replicas are balanced across regions, so one entire region can fail without interrupting access. + +Given that most of the data in your cluster is geo-partitioned, let's focus on AZ-level failure. + +1. SSH to the client VM in the US East region. + +2. Use the [`cockroach quit`](cockroach-quit.html) command to stop one node, effectively simulating one of the 3 AZ's failing: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=
+ ~~~ + +3. Back in the Admin UI, click **Overview** and note that the cluster now considers that node "suspect": + + Geo-partitioning resiliency + +4. Despite the node being unavailable, the MovR instance in US East continues to make progress because the other 2 nodes, each in a distinct AZ, remain available, and all partitions in the region therefore remain available. To verify this, click **Metrics**, select node 1 or 2 from the **Graph** menu, and check that SQL traffic continues on the node: + + Geo-partitioning resiliency + +## See also + +- Related Topology Patterns + - [Geo-Partitioned Replicas Topology](topology-geo-partitioned-replicas.html) + - [Duplicate Indexes Topology](topology-duplicate-indexes.html) + +- Related Case Studies + - [Electronic lock manufacturer](https://www.cockroachlabs.com/case-studies/european-electronic-lock-manufacturer-modernizes-iam-system-with-managed-cockroachdb/) + - [Multi-national bank](https://www.cockroachlabs.com/case-studies/top-five-multinational-bank-modernizes-its-european-core-banking-services-migrating-from-oracle-to-cockroachdb/) + +- [Reads and Writes in CockroachDB](architecture/reads-and-writes-overview.html) diff --git a/v20.2/demo-replication-and-rebalancing.md b/v20.2/demo-replication-and-rebalancing.md new file mode 100644 index 00000000000..519d3c8c969 --- /dev/null +++ b/v20.2/demo-replication-and-rebalancing.md @@ -0,0 +1,242 @@ +--- +title: Replication and Rebalancing +summary: Use a local cluster to explore how CockroachDB replicates and rebalances data. +toc: true +redirect_from: +- demo-automatic-rebalancing.html +- demo-data-replication.html +--- + +This page walks you through a simple demonstration of how CockroachDB replicates, distributes, and rebalances data. Starting with a 3-node local cluster, you'll write some data and verify that it replicates in triplicate by default. You'll then add 2 more nodes and watch how CockroachDB automatically rebalances replicas to efficiently use all available capacity. + +## Before you begin + +Make sure you have already [installed CockroachDB](install-cockroachdb.html). + +## Step 1. Start a 3-node cluster + +1. Use the [`cockroach start`](cockroach-start.html) command to start 3 nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=rep-node1 \ + --listen-addr=localhost:26257 \ + --http-addr=localhost:8080 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=rep-node2 \ + --listen-addr=localhost:26258 \ + --http-addr=localhost:8081 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --insecure \ + --store=rep-node3 \ + --listen-addr=localhost:26259 \ + --http-addr=localhost:8082 \ + --join=localhost:26257,localhost:26258,localhost:26259 \ + --background + ~~~ + +2. Use the [`cockroach init`](cockroach-init.html) command to perform a one-time initialization of the cluster: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach init \ + --insecure \ + --host=localhost:26257 + ~~~ + +## Step 2. Write data + +1. Use the [`cockroach workload`](cockroach-workload.html) command to generate an example `intro` database: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach workload init intro \ + 'postgresql://root@localhost:26257?sslmode=disable' + ~~~ + +2. Open the [built-in SQL shell](cockroach-sql.html) and verify that the new `intro` database was added with one table, `mytable`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost:26257 + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW DATABASES; + ~~~ + + ~~~ + database_name + +---------------+ + defaultdb + intro + postgres + system + (4 rows) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW TABLES FROM intro; + ~~~ + + ~~~ + table_name + +------------+ + mytable + (1 row) + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM intro.mytable WHERE (l % 2) = 0; + ~~~ + + ~~~ + l | v + +----+------------------------------------------------------+ + 0 | !__aaawwmqmqmwwwaas,,_ .__aaawwwmqmqmwwaaa,, + 2 | !"VT?!"""^~~^"""??T$Wmqaa,_auqmWBT?!"""^~~^^""??YV^ + 4 | ! "?##mW##?"- + 6 | ! C O N G R A T S _am#Z??A#ma, Y + 8 | ! _ummY" "9#ma, A + 10 | ! vm#Z( )Xmms Y + 12 | ! .j####mmm#####mm#m##6. + 14 | ! W O W ! jmm###mm######m#mmm##6 + 16 | ! ]#me*Xm#m#mm##m#m##SX##c + 18 | ! dm#||+*$##m#mm#m#Svvn##m + 20 | ! :mmE=|+||S##m##m#1nvnnX##; A + 22 | ! :m#h+|+++=Xmm#m#1nvnnvdmm; M + 24 | ! Y $#m>+|+|||##m#1nvnnnnmm# A + 26 | ! O ]##z+|+|+|3#mEnnnnvnd##f Z + 28 | ! U D 4##c|+|+|]m#kvnvnno##P E + 30 | ! I 4#ma+|++]mmhvnnvq##P` ! + 32 | ! D I ?$#q%+|dmmmvnnm##! + 34 | ! T -4##wu#mm#pw##7' + 36 | ! -?$##m####Y' + 38 | ! !! "Y##Y"- + 40 | ! + (21 rows) + ~~~ + +3. Exit the SQL shell: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +## Step 3. Verify replication + +1. To understand replication in CockroachDB, it's important to review a few concepts from the [architecture](architecture/overview.html): + + + Concept | Description + --------|------------ + **Range** | CockroachDB stores all user data (tables, indexes, etc.) and almost all system data in a giant sorted map of key-value pairs. This keyspace is divided into "ranges", contiguous chunks of the keyspace, so that every key can always be found in a single range.

From a SQL perspective, a table and its secondary indexes initially map to a single range, where each key-value pair in the range represents a single row in the table (also called the primary index because the table is sorted by the primary key) or a single row in a secondary index. As soon as that range reaches 512 MiB in size, it splits into two ranges. This process continues for these new ranges as the table and its indexes continue growing. + **Replica** | CockroachDB replicates each range (3 times by default) and stores each replica on a different node. + +2. With those concepts in mind, open the Admin UI at http://localhost:8080 and view the **Node List**: + + CockroachDB Admin UI + + Note that the **Replicas** count is the same on all three nodes. This indicates: + - There are this many "ranges" of data in the cluster. These are mostly internal "system" ranges since you haven't added much table data. + - Each range has been replicated 3 times (according to the CockroachDB default). + - For each range, each replica is stored on different nodes. + +## Step 4. Add two more nodes + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--store=rep-node4 \ +--listen-addr=localhost:26260 \ +--http-addr=localhost:8083 \ +--join=localhost:26257 \ +--background +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start \ +--insecure \ +--store=rep-node5 \ +--listen-addr=localhost:26261 \ +--http-addr=localhost:8084 \ +--join=localhost:26257 \ +--background +~~~ + +## Step 5. Watch data rebalance + +Back in the Admin UI, you'll see that there are now 5 nodes listed: + +CockroachDB Admin UI + +At first, the replica count will be lower for nodes 4 and 5. Very soon, however, you'll see those numbers even out across all nodes, indicating that data is being automatically rebalanced to utilize the additional capacity of the new nodes. + +## Step 6. Stop the cluster + +1. When you're done with your test cluster, use the [`cockroach quit`](cockroach-quit.html) command to gracefully shut down each node. + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26257 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26258 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26259 + ~~~ + + {{site.data.alerts.callout_info}} + For the last 2 nodes, the shutdown process will take longer (about a minute each) and will eventually force the nodes to stop. This is because, with only 2 of 5 nodes left, a majority of replicas are not available, and so the cluster is no longer operational. + {{site.data.alerts.end}} + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26260 + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach quit --insecure --host=localhost:26261 + ~~~ + +2. To restart the cluster at a later time, run the same `cockroach start` commands as earlier from the directory containing the nodes' data stores. + + If you do not plan to restart the cluster, you may want to remove the nodes' data stores: + + {% include copy-clipboard.html %} + ~~~ shell + $ rm -rf rep-node1 rep-node2 rep-node3 rep-node4 rep-node5 + ~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} diff --git a/v20.2/demo-serializable.md b/v20.2/demo-serializable.md new file mode 100644 index 00000000000..e420f1009d9 --- /dev/null +++ b/v20.2/demo-serializable.md @@ -0,0 +1,562 @@ +--- +title: Serializable Transactions +summary: +toc: true +--- + +In contrast to most databases, CockroachDB always uses `SERIALIZABLE` isolation, which is the strongest of the four [transaction isolation levels](https://en.wikipedia.org/wiki/Isolation_(database_systems)) defined by the SQL standard and is stronger than the `SNAPSHOT` isolation level developed later. `SERIALIZABLE` isolation guarantees that even though transactions may execute in parallel, the result is the same as if they had executed one at a time, without any concurrency. This ensures data correctness by preventing all "anomalies" allowed by weaker isolation levels. + +In this tutorial, you'll work through a hypothetical scenario that demonstrates the importance of `SERIALIZABLE` isolation for data correctness. + +1. You'll start by reviewing the scenario and its schema. +2. You'll then execute the scenario at one of the weaker isolation levels, `READ COMMITTED`, observing the write skew anomaly and its implications. Because CockroachDB always uses `SERIALIZABLE` isolation, you'll run this portion of the tutorial on Postgres, which defaults to `READ COMMITTED`. +3. You'll finish by executing the scenario at `SERIALIZABLE` isolation, observing how it guarantees correctness. You'll use CockroachDB for this portion. + +{{site.data.alerts.callout_info}} +For a deeper discussion of transaction isolation and the write skew anomaly, see the [Real Transactions are Serializable](https://www.cockroachlabs.com/blog/acid-rain/) and [What Write Skew Looks Like](https://www.cockroachlabs.com/blog/what-write-skew-looks-like/) blog posts. +{{site.data.alerts.end}} + +## Overview + +### Scenario + +- A hospital has an application for doctors to manage their on-call shifts. +- The hospital has a rule that at least one doctor must be on call at any one time. +- Two doctors are on-call for a particular shift, and both of them try to request leave for the shift at approximately the same time. +- In Postgres, with the default `READ COMMITTED` isolation level, the [write skew](#write-skew) anomaly results in both doctors successfully booking leave and the hospital having no doctors on call for that particular shift. +- In CockroachDB, with the `SERIALIZABLE` isolation level, write skew is prevented, one doctor is allowed to book leave and the other is left on-call, and lives are saved. + +#### Write skew + +When write skew happens, a transaction reads something, makes a decision based on the value it saw, and writes the decision to the database. However, by the time the write is made, the premise of the decision is no longer true. Only `SERIALIZABLE` and some implementations of `REPEATABLE READ` isolation prevent this anomaly. + +### Schema + +Schema for serializable transaction tutorial + +## Scenario on Postgres + +### Step 1. Start Postgres + +1. If you haven't already, install Postgres locally. On Mac, you can use [Homebrew](https://brew.sh/): + + {% include copy-clipboard.html %} + ~~~ shell + $ brew install postgres + ~~~ + +2. [Start Postgres](https://www.postgresql.org/docs/10/static/server-start.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ postgres -D /usr/local/var/postgres & + ~~~ + +### Step 2. Create the schema + +1. Open a SQL connection to Postgres: + + {% include copy-clipboard.html %} + ~~~ shell + $ psql + ~~~ + +2. Create the `doctors` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE doctors ( + id INT PRIMARY KEY, + name TEXT + ); + ~~~ + +3. Create the `schedules` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE schedules ( + day DATE, + doctor_id INT REFERENCES doctors (id), + on_call BOOL, + PRIMARY KEY (day, doctor_id) + ); + ~~~ + +### Step 3. Insert data + +1. Add two doctors to the `doctors` table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO doctors VALUES + (1, 'Abe'), + (2, 'Betty'); + ~~~ + +2. Insert one week's worth of data into the `schedules` table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO schedules VALUES + ('2018-10-01', 1, true), + ('2018-10-01', 2, true), + ('2018-10-02', 1, true), + ('2018-10-02', 2, true), + ('2018-10-03', 1, true), + ('2018-10-03', 2, true), + ('2018-10-04', 1, true), + ('2018-10-04', 2, true), + ('2018-10-05', 1, true), + ('2018-10-05', 2, true), + ('2018-10-06', 1, true), + ('2018-10-06', 2, true), + ('2018-10-07', 1, true), + ('2018-10-07', 2, true); + ~~~ + +3. Confirm that at least one doctor is on call each day of the week: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT day, count(*) AS doctors_on_call FROM schedules + WHERE on_call = true + GROUP BY day + ORDER BY day; + ~~~ + + ~~~ + day | doctors_on_call + ------------+----------------- + 2018-10-01 | 2 + 2018-10-02 | 2 + 2018-10-03 | 2 + 2018-10-04 | 2 + 2018-10-05 | 2 + 2018-10-06 | 2 + 2018-10-07 | 2 + (7 rows) + ~~~ + +### Step 4. Doctor 1 requests leave + +Doctor 1, Abe, starts to request leave for 10/5/18 using the hospital's schedule management application. + +1. The application starts a transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > BEGIN; + ~~~ + +2. The application checks to make sure at least one other doctor is on call for the requested date: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT count(*) FROM schedules + WHERE on_call = true + AND day = '2018-10-05' + AND doctor_id != 1; + ~~~ + + ~~~ + count + ------- + 1 + (1 row) + ~~~ + +### Step 5. Doctor 2 requests leave + +Around the same time, doctor 2, Betty, starts to request leave for the same day using the hospital's schedule management application. + +1. In a new terminal, start a second SQL session: + + {% include copy-clipboard.html %} + ~~~ shell + $ psql + ~~~ + +2. The application starts a transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > BEGIN; + ~~~ + +3. The application checks to make sure at least one other doctor is on call for the requested date: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT count(*) FROM schedules + WHERE on_call = true + AND day = '2018-10-05' + AND doctor_id != 2; + ~~~ + + ~~~ + count + ------- + 1 + (1 row) + ~~~ + +### Step 6. Leave is incorrectly booked for both doctors + +1. In the terminal for doctor 1, since the previous check confirmed that another doctor is on call for 10/5/18, the application tries to update doctor 1's schedule: + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE schedules SET on_call = false + WHERE day = '2018-10-05' + AND doctor_id = 1; + ~~~ + +2. In the terminal for doctor 2, since the previous check confirmed the same thing, the application tries to update doctor 2's schedule: + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE schedules SET on_call = false + WHERE day = '2018-10-05' + AND doctor_id = 2; + ~~~ + +3. In the terminal for doctor 1, the application commits the transaction, despite the fact that the previous check (the `SELECT` query) is no longer true: + + {% include copy-clipboard.html %} + ~~~ sql + > COMMIT; + ~~~ + +4. In the terminal for doctor 2, the application commits the transaction, despite the fact that the previous check (the `SELECT` query) is no longer true: + + {% include copy-clipboard.html %} + ~~~ sql + > COMMIT; + ~~~ + +### Step 7. Check data correctness + +So what just happened? Each transaction started by reading a value that, before the end of the transaction, became incorrect. Despite that fact, each transaction was allowed to commit. This is known as write skew, and the result is that 0 doctors are scheduled to be on call on 10/5/18. + +To check this, in either terminal, run: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM schedules WHERE day = '2018-10-05'; +~~~ + +~~~ + day | doctor_id | on_call +------------+-----------+--------- + 2018-10-05 | 1 | f + 2018-10-05 | 2 | f +(2 rows) +~~~ + +Again, this anomaly is the result of Postgres' default isolation level of `READ COMMITTED`, but note that this would happen with any isolation level except `SERIALIZABLE` and some implementations of `REPEATABLE READ`: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TRANSACTION_ISOLATION; +~~~ + +~~~ + transaction_isolation +----------------------- + read committed +(1 row) +~~~ + +### Step 8. Stop Postgres + +Exit each SQL shell with `\q` and then stop the Postgres server: + +{% include copy-clipboard.html %} +~~~ shell +$ pkill -9 postgres +~~~ + +## Scenario on CockroachDB + +When you repeat the scenario on CockroachDB, you'll see that the anomaly is prevented by CockroachDB's `SERIALIZABLE` transaction isolation. + +### Step 1. Start CockroachDB + +1. If you haven't already, [install CockroachDB](install-cockroachdb.html) locally. + +2. Use the [`cockroach start-single-node`](cockroach-start-single-node.html) command to start a one-node CockroachDB cluster in insecure mode: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start-single-node \ + --insecure \ + --store=serializable-demo \ + --listen-addr=localhost \ + --background + ~~~ + +### Step 2. Create the schema + +1. As the `root` user, open the [built-in SQL client](cockroach-sql.html): + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost + ~~~ + +2. Create the `doctors` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE doctors ( + id INT PRIMARY KEY, + name TEXT + ); + ~~~ + +3. Create the `schedules` table: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE schedules ( + day DATE, + doctor_id INT REFERENCES doctors (id), + on_call BOOL, + PRIMARY KEY (day, doctor_id) + ); + ~~~ + +### Step 3. Insert data + +1. Add two doctors to the `doctors` table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO doctors VALUES + (1, 'Abe'), + (2, 'Betty'); + ~~~ + +2. Insert one week's worth of data into the `schedules` table: + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO schedules VALUES + ('2018-10-01', 1, true), + ('2018-10-01', 2, true), + ('2018-10-02', 1, true), + ('2018-10-02', 2, true), + ('2018-10-03', 1, true), + ('2018-10-03', 2, true), + ('2018-10-04', 1, true), + ('2018-10-04', 2, true), + ('2018-10-05', 1, true), + ('2018-10-05', 2, true), + ('2018-10-06', 1, true), + ('2018-10-06', 2, true), + ('2018-10-07', 1, true), + ('2018-10-07', 2, true); + ~~~ + +3. Confirm that at least one doctor is on call each day of the week: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT day, count(*) AS on_call FROM schedules + WHERE on_call = true + GROUP BY day + ORDER BY day; + ~~~ + + ~~~ + day | on_call + +---------------------------+---------+ + 2018-10-01 00:00:00+00:00 | 2 + 2018-10-02 00:00:00+00:00 | 2 + 2018-10-03 00:00:00+00:00 | 2 + 2018-10-04 00:00:00+00:00 | 2 + 2018-10-05 00:00:00+00:00 | 2 + 2018-10-06 00:00:00+00:00 | 2 + 2018-10-07 00:00:00+00:00 | 2 + (7 rows) + ~~~ + +### Step 4. Doctor 1 requests leave + +Doctor 1, Abe, starts to request leave for 10/5/18 using the hospital's schedule management application. + +1. The application starts a transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > BEGIN; + ~~~ + +2. The application checks to make sure at least one other doctor is on call for the requested date: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT count(*) FROM schedules + WHERE on_call = true + AND day = '2018-10-05' + AND doctor_id != 1; + ~~~ + + Press enter a second time to have the server return the result: + + ~~~ + count + +-------+ + 1 + (1 row) + ~~~ + +### Step 5. Doctor 2 requests leave + +Around the same time, doctor 2, Betty, starts to request leave for the same day using the hospital's schedule management application. + +1. In a new terminal, start a second SQL session: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --insecure --host=localhost + ~~~ + +2. The application starts a transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > BEGIN; + ~~~ + +3. The application checks to make sure at least one other doctor is on call for the requested date: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT count(*) FROM schedules + WHERE on_call = true + AND day = '2018-10-05' + AND doctor_id != 2; + ~~~ + + Press enter a second time to have the server return the result: + + ~~~ + count + +-------+ + 1 + (1 row) + ~~~ + +### Step 6. Leave is booked for only 1 doctor + +1. In the terminal for doctor 1, since the previous check confirmed that another doctor is on call for 10/5/18, the application tries to update doctor 1's schedule: + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE schedules SET on_call = false + WHERE day = '2018-10-05' + AND doctor_id = 1; + ~~~ + +2. In the terminal for doctor 2, since the previous check confirmed the same thing, the application tries to update doctor 2's schedule: + + {% include copy-clipboard.html %} + ~~~ sql + > UPDATE schedules SET on_call = false + WHERE day = '2018-10-05' + AND doctor_id = 2; + ~~~ + +3. In the terminal for doctor 1, the application tries to commit the transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > COMMIT; + ~~~ + + Since CockroachDB uses `SERIALIZABLE` isolation, the database detects that the previous check (the `SELECT` query) is no longer true due to a concurrent transaction. It therefore prevents the transaction from committing, returning a retry error that indicates that the transaction must be attempted again: + + ~~~ + pq: restart transaction: TransactionRetryWithProtoRefreshError: TransactionRetryError: retry txn (RETRY_SERIALIZABLE): id=373bbefe key=/Table/53/1/17809/1/0 rw=true pri=0.03885012 stat=PENDING epo=0 ts=1569638527.268184000,1 orig=1569638507.593587000,0 min=1569638507.593587000,0 max=1569638507.593587000,0 wto=false seq=2 + ~~~ + + {{site.data.alerts.callout_success}} + For this kind of error, CockroachDB recommends a [client-side transaction retry loop](transactions.html#client-side-intervention) that would transparently observe that the one doctor cannot take time off because the other doctor already succeeded in asking for it. You can find generic transaction retry functions for various languages in our [Build an App](build-an-app-with-cockroachdb.html) tutorials. + {{site.data.alerts.end}} + +4. In the terminal for doctor 2, the application tries to commit the transaction: + + {% include copy-clipboard.html %} + ~~~ sql + > COMMIT; + ~~~ + + Since the transaction for doctor 1 failed, the transaction for doctor 2 can commit without causing any data correctness problems. + +### Step 7. Check data correctness + +1. In either terminal, confirm that one doctor is still on call for 10/5/18: + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM schedules WHERE day = '2018-10-05'; + ~~~ + + ~~~ + day | doctor_id | on_call + +---------------------------+-----------+---------+ + 2018-10-05 00:00:00+00:00 | 1 | true + 2018-10-05 00:00:00+00:00 | 2 | false + (2 rows) + ~~~ + +2. Again, the write skew anomaly was prevented by CockroachDB using the `SERIALIZABLE` isolation level: + + {% include copy-clipboard.html %} + ~~~ sql + > SHOW TRANSACTION_ISOLATION; + ~~~ + + ~~~ + transaction_isolation + +-----------------------+ + serializable + (1 row) + ~~~ + +3. Exit the SQL shell in each terminal: + + {% include copy-clipboard.html %} + ~~~ sql + > \q + ~~~ + +### Step 8. Stop CockroachDB + +Once you're done with your test cluster, exit each SQL shell with `\q` and then stop the node: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach quit --insecure --host=localhost +~~~ + +If you do not plan to restart the cluster, you may want to remove the node's data store: + +{% include copy-clipboard.html %} +~~~ shell +$ rm -rf serializable-demo +~~~ + +## What's next? + +Explore other core CockroachDB benefits and features: + +{% include {{ page.version.version }}/misc/explore-benefits-see-also.md %} + +You might also want to learn more about how transactions work in CockroachDB and in general: + +- [Transactions Overview](transactions.html) +- [Real Transactions are Serializable](https://www.cockroachlabs.com/blog/acid-rain/) +- [What Write Skew Looks Like](https://www.cockroachlabs.com/blog/what-write-skew-looks-like/) diff --git a/v20.2/deploy-a-test-cluster.md b/v20.2/deploy-a-test-cluster.md new file mode 100644 index 00000000000..93489a8dfd6 --- /dev/null +++ b/v20.2/deploy-a-test-cluster.md @@ -0,0 +1,168 @@ +--- +title: Deploy a Test Cluster +summary: Use CockroachDB's CloudFormation template to deploy a Kubernetes-orchestrated test cluster on AWS. +toc: true +--- + +This page shows you the easiest way to test an insecure, multi-node CockroachDB cluster, using CockroachDB's [AWS CloudFormation](https://aws.amazon.com/cloudformation/) template to simplify setup and [Kubernetes](https://kubernetes.io/) to automate deployment, maintenance, and load balancing of client workloads. + + + + +## Before you begin + +Before getting started, it's important to review some limitations and requirements. + +### Limitations + +{{site.data.alerts.callout_danger}}The CockroachDB AWS CloudFormation template is designed for testing, not for production use.{{site.data.alerts.end}} + +- You can scale the cluster to a maximum of 15 nodes. + +- While the AWS region for your deployment is configurable, the cluster runs in a single AWS availability zone within that region. It will easily survive and recover from node failures as long as you deploy at least 3 nodes, but it will not survive an availability zone outage. + - For production resiliency, the recommendation would be to span 3 or more availability zones in a single region or 3 or more regions. + +- The cluster is completely insecure, which comes with risks: + - There is no network encryption or authentication, and thus no confidentiality. + - The cluster is open to any client by default, although you have the option to restrict client access to a specific range of IP addresses. + - Any user, even `root`, can log in without providing a password. + - Any user, connecting as `root`, can read or write any data in your cluster. + +### Requirements + +- You must have an [AWS account](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-sign-up-for-aws.html). +- You must have [SSH access](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html) in the AWS region where the cluster is deployed. + +## Step 1. Start CockroachDB + +1. [Launch the CockroachDB CloudFormation template](http://amzn.to/2CZjJLZ). + +2. In the CloudFormation UI, review and customize the settings for the cluster. Most of the defaults are sufficient for testing scenarios. However, it's important to select your **SSH Key** so you'll be able to connect to the Kubernetes master node later, and to set **CockroachDB Version** to the **v2.0** option. + + You may also want to: + - Change the **AWS region** where the cluster will run. The default region is **US West**. Note that some instance types may not be available in some regions. + - Add an **IP Address Whitelist** to restrict user access to the CockroachDB Admin UI and SQL client access to the cluster. By default, all locations have access. + - Increase the initial **Cluster Size**. The default is **3** nodes. + +3. In the **Load Generators** section, select the type of **Workload** you would like to run against the cluster. + +4. When you're ready to start the cluster, click **Create**. + + The launch process generally takes 10 to 15 minutes. Once you see the `CREATE_COMPLETE` status in the CloudFormation UI, the cluster is ready for testing. + + {{site.data.alerts.callout_info}}If the launch process times out or fails, you could be running into an AWS service limit. You can view any errors in the event history.{{site.data.alerts.end}} + +## Step 2. Test the cluster + +1. [Install CockroachDB](install-cockroachdb.html) on your local machine, if you haven't already. + +2. In the **Outputs** section of the CloudFormation UI, note the **Connection String**. + +3. In a terminal, start the [SQL shell](cockroach-sql.html) built into the `cockroach` binary, using the **Connection String** as the `--url` flag: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql \ + --insecure \ + --url="postgresql://root@Cockroach-ApiLoadB-LVZZ3VVHMIDA-1266691548.us-west-2.elb.amazonaws.com:26257?sslmode=disable" + ~~~ + +4. Run some basic [CockroachDB SQL statements](learn-cockroachdb-sql.html): + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE DATABASE bank; + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE TABLE bank.accounts (id INT PRIMARY KEY, balance DECIMAL); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > INSERT INTO bank.accounts VALUES (1, 1000.50); + ~~~ + + {% include copy-clipboard.html %} + ~~~ sql + > SELECT * FROM bank.accounts; + ~~~ + + ~~~ + +----+---------+ + | id | balance | + +----+---------+ + | 1 | 1000.5 | + +----+---------+ + (1 row) + ~~~ + +{{site.data.alerts.callout_success}}With the cockroach binary on your local machine, other client cockroach commands can be run in the same way.{{site.data.alerts.end}} + +## Step 3. Monitor the cluster + +You can use the cluster's [Admin UI](admin-ui-overview.html) to monitor the workload and overall cluster behavior. + +1. In the **Outputs** section of the CloudFormation UI, click the **Web UI** link. Then click **Metrics** on the left-hand navigation bar. + +2. On the **Overview** dashboard, hover over the **SQL Queries** graph to see the proportion of reads and writes coming from the load generator. + + CockroachDB Admin UI + +3. Scroll down and hover over the **Replicas per Node** graph to see how CockroachDB automatically replicates your data behind-the-scenes. + + CockroachDB Admin UI + +4. Explore other areas of the [Admin UI](admin-ui-overview.html). + +5. Learn more about [production monitoring and alerting](monitoring-and-alerting.html). + +## Step 4. Simulate node failure + +Kubernetes ensures that the cluster always has the number of nodes you specified during initial configuration (3 by default). When a node fails, Kubernetes automatically creates another node with the same network identity and persistent storage. + +To see this in action: + +1. In the **Outputs** section of the CloudFormation UI, note the **SSHProxyCommand**. + +2. In a new terminal, run the **SSHProxyCommand** to SSH into the Kubernetes master node. Be sure to update the `SSH_KEY` environment variable definition to point to the location of your `.pem` file. + +3. List the Kubernetes pods that map to CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl get pods + ~~~ + + ~~~ + NAME READY STATUS RESTARTS AGE + cockroachdb-0 1/1 Running 0 1h + cockroachdb-1 1/1 Running 0 1h + cockroachdb-2 1/1 Running 0 1h + ~~~ + +3. Kill one of CockroachDB nodes: + + {% include copy-clipboard.html %} + ~~~ shell + $ kubectl delete pod cockroachdb-2 + ~~~ + + ~~~ + pod "cockroachdb-2" deleted + ~~~ + +4. In the Admin UI, the **Cluster Overview** panel may show one node as **Suspect**. As Kubernetes auto-restarts the node, watch how the node once again becomes healthy. + + You can also select the **Runtime** dashboard and see the restarting of the node in the **Live Node Count** graph. + + CockroachDB Admin UI + +## Step 6. Stop the cluster + +In the CloudFormation UI, select **Other Actions** > **Delete Stack**. This is essential for deleting all AWS resources tied to your cluster. If you do not delete these resources, AWS will continue to charge you for them. + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-aws-insecure.md b/v20.2/deploy-cockroachdb-on-aws-insecure.md new file mode 100644 index 00000000000..593b742dc46 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-aws-insecure.md @@ -0,0 +1,164 @@ +--- +title: Deploy CockroachDB on AWS EC2 (Insecure) +summary: Learn how to deploy CockroachDB on Amazon's AWS EC2 platform. +toc: true +toc_not_nested: true +ssh-link: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html +--- + + + +This page shows you how to manually deploy an insecure multi-node CockroachDB cluster on Amazon's AWS EC2 platform, using AWS's managed load balancing service to distribute client traffic. + +{{site.data.alerts.callout_danger}}If you plan to use CockroachDB in production, we strongly recommend using a secure cluster instead. Select Secure above for instructions.{{site.data.alerts.end}} + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/insecure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/insecure-recommendations.md %} + +- All Amazon EC2 instances running CockroachDB should be members of the same [security group](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html). + +## Step 1. Create instances + +Open the [Amazon EC2 console](https://console.aws.amazon.com/ec2/) and [launch an instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/launching-instance.html#launch-instance-console) for each node you plan to have in your cluster. If you plan to [run our sample workload](#step-8-run-a-sample-workload) against the cluster, create a separate instance for that workload. + +- Run at least 3 nodes to ensure survivability. + +- Your instances will rely on Amazon Time Sync Service for clock synchronization. When choosing an AMI, note that some machines are preconfigured to use Amazon Time Sync Service (e.g., Amazon Linux AMIs) and others are not. + +- Use `m` (general purpose), `c` (compute-optimized), or `i` (storage-optimized) instance types, with SSD-backed [EBS volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) or [Instance Store volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ssd-instance-store.html). For example, Cockroach Labs has used `c5d.4xlarge` (16 vCPUs and 32 GiB of RAM per instance, EBS) for internal testing. + + - **Do not** use ["burstable" `t2` instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/t2-instances.html), which limit the load on a single core. + +- Note the ID of the VPC you select. You will need to look up its IP range when setting inbound rules for your security group. + +- Make sure all your instances are in the same security group. + + - If you are creating a new security group, add the [inbound rules](#step-2-configure-your-network) from the next step. Otherwise note the ID of the security group. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 2. Configure your network + +CockroachDB requires TCP communication on two ports: + +- `26257` for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- `8080` for exposing your Admin UI, and for routing from the load balancer to the health check + +If you haven't already done so, [create inbound rules](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html#adding-security-group-rule) for your security group. + +#### Inter-node and load balancer-node communication + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **26257** + Source | The ID of your security group (e.g., *sg-07ab277a*) + +#### Application data + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rules + Protocol | TCP + Port Range | **26257** + Source | Your application's IP ranges + +If you plan to [run our sample workload](#step-8-run-a-sample-workload) on an instance, the traffic source is the internal (private) IP address of that instance. To find this, open the Instances section of the Amazon EC2 console and click on the instance. + +#### Admin UI + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **8080** + Source | Your network's IP ranges + +You can set your network IP by selecting "My IP" in the Source field. + +#### Load balancer-health check communication + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **8080** + Source | The IP range of your VPC in CIDR notation (e.g., 10.12.0.0/16) + +To get the IP range of a VPC, open the [Amazon VPC console](https://console.aws.amazon.com/vpc/) and find the VPC listed in the section called Your VPCs. You can also click on the VPC where it is listed in the EC2 console. + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +AWS offers fully-managed load balancing to distribute traffic between instances. + +1. [Add AWS load balancing](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancer-getting-started.html). Be sure to: + - Select a **Network Load Balancer** and use the ports we specify below. + - Select the VPC and *all* availability zones of your instances. This is important, as you cannot change the availability zones once the load balancer is created. The availability zone of an instance is determined by its subnet, found by inspecting the instance in the Amazon EC2 Console. + - Set the load balancer port to **26257**. + - Create a new target group that uses TCP port **26257**. Traffic from your load balancer is routed to this target group, which contains your instances. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + - Register your instances with the target group you created, specifying port **26257**. You can add and remove instances later. +2. To test load balancing and connect your application to the cluster, you will need the provisioned internal (private) **IP address** for the load balancer. To find this, open the Network Interfaces section of the Amazon EC2 console and look up the load balancer by its name. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of AWS's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 5. Start nodes + +{% include {{ page.version.version }}/prod-deployment/insecure-start-nodes.md %} + +## Step 6. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-initialize-cluster.md %} + +## Step 7. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-test-cluster.md %} + +## Step 8. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/insecure-test-load-balancing.md %} + +## Step 9. Monitor the cluster + +In the Target Groups section of the Amazon EC2 console, [check the health](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/target-group-health-checks.html) of your instances by inspecting your target group and opening the Targets tab. + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 10. Scale the cluster + +Before adding a new node, [create a new instance](#step-1-create-instances) as you did earlier. + +{% include {{ page.version.version }}/prod-deployment/insecure-scale-cluster.md %} + +## Step 11. Use the cluster + +Now that your deployment is working, you can: + +1. [Implement your data model](sql-statements.html). +2. [Create users](create-and-manage-users.html) and [grant them privileges](grant.html). +3. [Connect your application](install-client-drivers.html). Be sure to connect your application to the AWS load balancer, not to a CockroachDB node. + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-aws.md b/v20.2/deploy-cockroachdb-on-aws.md new file mode 100644 index 00000000000..1c234776b97 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-aws.md @@ -0,0 +1,167 @@ +--- +title: Deploy CockroachDB on AWS EC2 +summary: Learn how to deploy CockroachDB on Amazon's AWS EC2 platform. +toc: true +toc_not_nested: true +ssh-link: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html + +--- + +
+ + +
+ +This page shows you how to manually deploy a secure multi-node CockroachDB cluster on Amazon's AWS EC2 platform, using AWS's managed load balancing service to distribute client traffic. + +If you are only testing CockroachDB, or you are not concerned with protecting network communication with TLS encryption, you can use an insecure cluster instead. Select **Insecure** above for instructions. + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/secure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/secure-recommendations.md %} + +- All Amazon EC2 instances running CockroachDB should be members of the same [security group](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html). + +## Step 1. Create instances + +Open the [Amazon EC2 console](https://console.aws.amazon.com/ec2/) and [launch an instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/launching-instance.html#launch-instance-console) for each node you plan to have in your cluster. If you plan to [run our sample workload](#step-9-run-a-sample-workload) against the cluster, create a separate instance for that workload. + +- Run at least 3 nodes to ensure survivability. + +- Your instances will rely on Amazon Time Sync Service for clock synchronization. When choosing an AMI, note that some machines are preconfigured to use Amazon Time Sync Service (e.g., Amazon Linux AMIs) and others are not. + +- Use `m` (general purpose), `c` (compute-optimized), or `i` (storage-optimized) instance types, with SSD-backed [EBS volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) or [Instance Store volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ssd-instance-store.html). For example, Cockroach Labs has used `c5d.4xlarge` (16 vCPUs and 32 GiB of RAM per instance, EBS) for internal testing. + + - **Do not** use ["burstable" `t2` instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/t2-instances.html), which limit the load on a single core. + +- Note the ID of the VPC you select. You will need to look up its IP range when setting inbound rules for your security group. + +- Make sure all your instances are in the same security group. + + - If you are creating a new security group, add the [inbound rules](#step-2-configure-your-network) from the next step. Otherwise note the ID of the security group. + +- When creating the instance, you will download a private key file used to securely connect to your instances. Decide where to place this file, and note the file path for later commands. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 2. Configure your network + +CockroachDB requires TCP communication on two ports: + +- `26257` for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- `8080` for exposing your Admin UI, and for routing from the load balancer to the health check + +If you haven't already done so, [create inbound rules](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-network-security.html#adding-security-group-rule) for your security group. + +#### Inter-node and load balancer-node communication + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **26257** + Source | The ID of your security group (e.g., *sg-07ab277a*) + +#### Application data + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rules + Protocol | TCP + Port Range | **26257** + Source | Your application's IP ranges + +If you plan to [run our sample workload](#step-9-run-a-sample-workload) on an instance, the traffic source is the internal (private) IP address of that instance. To find this, open the Instances section of the Amazon EC2 console and click on the instance. + +#### Admin UI + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **8080** + Source | Your network's IP ranges + +You can set your network IP by selecting "My IP" in the Source field. + +#### Load balancer-health check communication + + Field | Recommended Value +-------|------------------- + Type | Custom TCP Rule + Protocol | TCP + Port Range | **8080** + Source | The IP range of your VPC in CIDR notation (e.g., 10.12.0.0/16) + +To get the IP range of a VPC, open the [Amazon VPC console](https://console.aws.amazon.com/vpc/) and find the VPC listed in the section called Your VPCs. You can also click on the VPC where it is listed in the EC2 console. + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +AWS offers fully-managed load balancing to distribute traffic between instances. + +1. [Add AWS load balancing](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancer-getting-started.html). Be sure to: + - Select a **Network Load Balancer** and use the ports we specify below. + - Select the VPC and *all* availability zones of your instances. This is important, as you cannot change the availability zones once the load balancer is created. The availability zone of an instance is determined by its subnet, found by inspecting the instance in the Amazon EC2 Console. + - Set the load balancer port to **26257**. + - Create a new target group that uses TCP port **26257**. Traffic from your load balancer is routed to this target group, which contains your instances. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + - Register your instances with the target group you created, specifying port **26257**. You can add and remove instances later. +2. To test load balancing and connect your application to the cluster, you will need the provisioned internal (private) **IP address** for the load balancer. To find this, open the Network Interfaces section of the Amazon EC2 console and look up the load balancer by its name. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of AWS's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 5. Generate certificates + +{% include {{ page.version.version }}/prod-deployment/secure-generate-certificates.md %} + +## Step 6. Start nodes + +{% include {{ page.version.version }}/prod-deployment/secure-start-nodes.md %} + +## Step 7. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-initialize-cluster.md %} + +## Step 8. Test your cluster + +{% include {{ page.version.version }}/prod-deployment/secure-test-cluster.md %} + +## Step 9. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/secure-test-load-balancing.md %} + +## Step 10. Monitor the cluster + +In the Target Groups section of the Amazon EC2 console, [check the health](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/target-group-health-checks.html) of your instances by inspecting your target group and opening the Targets tab. + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 11. Scale the cluster + +Before adding a new node, [create a new instance](#step-1-create-instances) as you did earlier. Then [generate and upload a certificate and key](#step-5-generate-certificates) for the new node. + +{% include {{ page.version.version }}/prod-deployment/secure-scale-cluster.md %} + +## Step 12. Use the database + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-digital-ocean-insecure.md b/v20.2/deploy-cockroachdb-on-digital-ocean-insecure.md new file mode 100644 index 00000000000..95163c2218c --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-digital-ocean-insecure.md @@ -0,0 +1,110 @@ +--- +title: Deploy CockroachDB on Digital Ocean (Insecure) +summary: Learn how to deploy CockroachDB on Digital Ocean. +toc: true +toc_not_nested: true +ssh-link: https://www.digitalocean.com/community/tutorials/how-to-connect-to-your-droplet-with-ssh +--- + + + +This page shows you how to deploy an insecure multi-node CockroachDB cluster on Digital Ocean, using Digital Ocean's managed load balancing service to distribute client traffic. + +{{site.data.alerts.callout_danger}}If you plan to use CockroachDB in production, we strongly recommend using a secure cluster instead. Select Secure above for instructions.{{site.data.alerts.end}} + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/insecure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/insecure-recommendations.md %} + +- If all of your CockroachDB nodes and clients will run on Droplets in a single region, consider using [private networking](https://www.digitalocean.com/community/tutorials/how-to-set-up-and-use-digitalocean-private-networking). + +## Step 1. Create Droplets + +[Create Droplets](https://www.digitalocean.com/community/tutorials/how-to-create-your-first-digitalocean-droplet) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate droplet for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use any [droplets](https://www.digitalocean.com/pricing/) except standard droplets with only 1 GB of RAM, which is below our minimum requirement. All Digital Ocean droplets use SSD storage. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 2. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 3. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +Digital Ocean offers fully-managed load balancers to distribute traffic between Droplets. + +1. [Create a Digital Ocean Load Balancer](https://www.digitalocean.com/community/tutorials/an-introduction-to-digitalocean-load-balancers). Be sure to: + - Set forwarding rules to route TCP traffic from the load balancer's port **26257** to port **26257** on the node Droplets. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. +2. Note the provisioned **IP Address** for the load balancer. You'll use this later to test load balancing and to connect your application to the cluster. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of Digital Ocean's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 4. Configure your network + +Set up a firewall for each of your Droplets, allowing TCP communication on the following two ports: + +- **26257** (`tcp:26257`) for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- **8080** (`tcp:8080`) for exposing your Admin UI + +For guidance, you can use Digital Ocean's guide to configuring firewalls based on the Droplet's OS: + +- Ubuntu and Debian can use [`ufw`](https://www.digitalocean.com/community/tutorials/how-to-setup-a-firewall-with-ufw-on-an-ubuntu-and-debian-cloud-server). +- FreeBSD can use [`ipfw`](https://www.digitalocean.com/community/tutorials/recommended-steps-for-new-freebsd-10-1-servers). +- Fedora can use [`iptables`](https://www.digitalocean.com/community/tutorials/initial-setup-of-a-fedora-22-server). +- CoreOS can use [`iptables`](https://www.digitalocean.com/community/tutorials/how-to-secure-your-coreos-cluster-with-tls-ssl-and-firewall-rules). +- CentOS can use [`firewalld`](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-firewall-using-firewalld-on-centos-7). + +## Step 5. Start nodes + +{% include {{ page.version.version }}/prod-deployment/insecure-start-nodes.md %} + +## Step 6. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-initialize-cluster.md %} + +## Step 7. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-test-cluster.md %} + +## Step 8. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/insecure-test-load-balancing.md %} + +## Step 9. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 10. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-scale-cluster.md %} + +## Step 11. Use the cluster + +Now that your deployment is working, you can: + +1. [Implement your data model](sql-statements.html). +2. [Create users](create-and-manage-users.html) and [grant them privileges](grant.html). +3. [Connect your application](install-client-drivers.html). Be sure to connect your application to the Digital Ocean Load Balancer, not to a CockroachDB node. + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-digital-ocean.md b/v20.2/deploy-cockroachdb-on-digital-ocean.md new file mode 100644 index 00000000000..b7aa65df3b8 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-digital-ocean.md @@ -0,0 +1,110 @@ +--- +title: Deploy CockroachDB on Digital Ocean +summary: Learn how to deploy CockroachDB on Digital Ocean. +toc: true +toc_not_nested: true +ssh-link: https://www.digitalocean.com/community/tutorials/how-to-connect-to-your-droplet-with-ssh +--- + +
+ + +
+ +This page shows you how to deploy a secure multi-node CockroachDB cluster on Digital Ocean, using Digital Ocean's managed load balancing service to distribute client traffic. + +If you are only testing CockroachDB, or you are not concerned with protecting network communication with TLS encryption, you can use an insecure cluster instead. Select **Insecure** above for instructions. + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/secure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/secure-recommendations.md %} + +- If all of your CockroachDB nodes and clients will run on Droplets in a single region, consider using [private networking](https://www.digitalocean.com/community/tutorials/how-to-set-up-and-use-digitalocean-private-networking). + +## Step 1. Create Droplets + +[Create Droplets](https://www.digitalocean.com/community/tutorials/how-to-create-your-first-digitalocean-droplet) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate Droplet for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use any [droplets](https://www.digitalocean.com/pricing/) except standard droplets with only 1 GB of RAM, which is below our minimum requirement. All Digital Ocean droplets use SSD storage. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 2. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 3. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +Digital Ocean offers fully-managed load balancers to distribute traffic between Droplets. + +1. [Create a Digital Ocean Load Balancer](https://www.digitalocean.com/community/tutorials/an-introduction-to-digitalocean-load-balancers). Be sure to: + - Set forwarding rules to route TCP traffic from the load balancer's port **26257** to port **26257** on the node Droplets. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. +2. Note the provisioned **IP Address** for the load balancer. You'll use this later to test load balancing and to connect your application to the cluster. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of Digital Ocean's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 4. Configure your network + +Set up a firewall for each of your Droplets, allowing TCP communication on the following two ports: + +- **26257** (`tcp:26257`) for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- **8080** (`tcp:8080`) for exposing your Admin UI + +For guidance, you can use Digital Ocean's guide to configuring firewalls based on the Droplet's OS: + +- Ubuntu and Debian can use [`ufw`](https://www.digitalocean.com/community/tutorials/how-to-setup-a-firewall-with-ufw-on-an-ubuntu-and-debian-cloud-server). +- FreeBSD can use [`ipfw`](https://www.digitalocean.com/community/tutorials/recommended-steps-for-new-freebsd-10-1-servers). +- Fedora can use [`iptables`](https://www.digitalocean.com/community/tutorials/initial-setup-of-a-fedora-22-server). +- CoreOS can use [`iptables`](https://www.digitalocean.com/community/tutorials/how-to-secure-your-coreos-cluster-with-tls-ssl-and-firewall-rules). +- CentOS can use [`firewalld`](https://www.digitalocean.com/community/tutorials/how-to-set-up-a-firewall-using-firewalld-on-centos-7). + +## Step 5. Generate certificates + +{% include {{ page.version.version }}/prod-deployment/secure-generate-certificates.md %} + +## Step 6. Start nodes + +{% include {{ page.version.version }}/prod-deployment/secure-start-nodes.md %} + +## Step 7. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-initialize-cluster.md %} + +## Step 8. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-test-cluster.md %} + +## Step 9. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/secure-test-load-balancing.md %} + +## Step 10. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 11. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-scale-cluster.md %} + +## Step 12. Use the database + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-google-cloud-platform-insecure.md b/v20.2/deploy-cockroachdb-on-google-cloud-platform-insecure.md new file mode 100644 index 00000000000..4169dab5ecf --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-google-cloud-platform-insecure.md @@ -0,0 +1,135 @@ +--- +title: Deploy CockroachDB on Google Cloud Platform GCE (Insecure) +summary: Learn how to deploy CockroachDB on Google Cloud Platform's Compute Engine. +toc: true +toc_not_nested: true +ssh-link: https://cloud.google.com/compute/docs/instances/connecting-to-instance +--- + + + +This page shows you how to manually deploy an insecure multi-node CockroachDB cluster on Google Cloud Platform's Compute Engine (GCE), using Google's TCP Proxy Load Balancing service to distribute client traffic. + +{{site.data.alerts.callout_danger}}If you plan to use CockroachDB in production, we strongly recommend using a secure cluster instead. Select Secure above for instructions.{{site.data.alerts.end}} + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/insecure-requirements.md %} + +- This article covers the use of Linux instances with GCE. You may wish to review the instructions for [connecting to Windows instances](https://cloud.google.com/compute/docs/instances/connecting-to-instance#windows). + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/insecure-recommendations.md %} + +## Step 1. Configure your network + +CockroachDB requires TCP communication on two ports: + +- `26257` for inter-node communication (i.e., working as a cluster) +- `8080` for exposing your Admin UI + +To expose your Admin UI and allow traffic from the TCP proxy load balancer and health checker to your instances, [create firewall rules](https://cloud.google.com/compute/docs/vpc/firewalls) for your project. When creating firewall rules, we recommend using Google Cloud Platform's **tag** feature to apply the rule only to instances with the same tag. + +#### Admin UI + + Field | Recommended Value +-------|------------------- + Name | **cockroachadmin** + Source filter | IP ranges + Source IP ranges | Your local network's IP ranges + Allowed protocols... | **tcp:8080** + Target tags | **cockroachdb** + +#### Application data + +Applications will not connect directly to your CockroachDB nodes. Instead, they'll connect to GCE's TCP Proxy Load Balancing service, which automatically routes traffic to the instances that are closest to the user. Because this service is implemented at the edge of the Google Cloud, you'll need to create a firewall rule to allow traffic from the load balancer and health checker to your instances. This is covered in [Step 4](#step-4-set-up-load-balancing). + +## Step 2. Create instances + +[Create an instance](https://cloud.google.com/compute/docs/instances/create-start-instance) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate instance for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use `n1-standard` or `n1-highcpu` [predefined VMs](https://cloud.google.com/compute/pricing#predefined_machine_types), or [custom VMs](https://cloud.google.com/compute/pricing#custommachinetypepricing), with [Local SSDs](https://cloud.google.com/compute/docs/disks/#localssds) or [SSD persistent disks](https://cloud.google.com/compute/docs/disks/#pdspecs). For example, Cockroach Labs has used `n1-standard-16` (16 vCPUs and 60 GB of RAM per VM, local SSD) for internal testing. + +- **Do not** use `f1` or `g1` [shared-core machines](https://cloud.google.com/compute/docs/machine-types#sharedcore), which limit the load on a single core. + +- If you used a tag for your firewall rules, when you create the instance, click **Management, security, disks, networking, sole tenancy**. Under the **Networking** tab, in the **Network tags** field, enter **cockroachdb**. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +GCE offers fully-managed [TCP Proxy Load Balancing](https://cloud.google.com/load-balancing/docs/tcp/). This service lets you use a single IP address for all users around the world, automatically routing traffic to the instances that are closest to the user. + +{{site.data.alerts.callout_danger}} +When using TCP Proxy Load Balancing, you cannot use firewall rules to control access to the load balancer. If you need such control, consider using [Network TCP Load Balancing](https://cloud.google.com/compute/docs/load-balancing/network/) instead, but note that it cannot be used across regions. You might also consider using the HAProxy load balancer (see the [On-Premises](deploy-cockroachdb-on-premises-insecure.html) tutorial for guidance). +{{site.data.alerts.end}} + +To use GCE's TCP Proxy Load Balancing service: + +1. For each zone in which you're running an instance, [create a distinct instance group](https://cloud.google.com/compute/docs/instance-groups/creating-groups-of-unmanaged-instances). + - To ensure that the load balancer knows where to direct traffic, specify a port name mapping, with `tcp26257` as the **Port name** and `26257` as the **Port number**. +2. [Add the relevant instances to each instance group](https://cloud.google.com/compute/docs/instance-groups/creating-groups-of-unmanaged-instances#addinstances). +3. [Configure Proxy Load Balancing](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#configure_load_balancer). + - During backend configuration, create a health check, setting the **Protocol** to `HTTP`, the **Port** to `8080`, and the **Request path** to path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + - If you want to maintain long-lived SQL connections that may be idle for more than tens of seconds, increase the backend timeout setting accordingly. + - During frontend configuration, reserve a static IP address and choose a port. Note this address/port combination, as you'll use it for all of you client connections. +4. [Create a firewall rule](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#config-hc-firewall) to allow traffic from the load balancer and health checker to your instances. This is necessary because TCP Proxy Load Balancing is implemented at the edge of the Google Cloud. + - Be sure to set **Source IP ranges** to `130.211.0.0/22` and `35.191.0.0/16` and set **Target tags** to `cockroachdb` (not to the value specified in the linked instructions). + +## Step 5. Start nodes + +{{site.data.alerts.callout_info}} +By default, inter-node communication uses the internal IP addresses of your GCE instances. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/prod-deployment/insecure-start-nodes.md %} + +## Step 6. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-initialize-cluster.md %} + +## Step 7. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-test-cluster.md %} + +## Step 8. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/insecure-test-load-balancing.md %} + +## Step 9. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 10. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-scale-cluster.md %} + +## Step 11. Use the cluster + +Now that your deployment is working, you can: + +1. [Implement your data model](sql-statements.html). +2. [Create users](create-user.html) and [grant them privileges](grant.html). +3. [Connect your application](install-client-drivers.html). Be sure to connect your application to the GCE load balancer, not to a CockroachDB node. + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-google-cloud-platform.md b/v20.2/deploy-cockroachdb-on-google-cloud-platform.md new file mode 100644 index 00000000000..408a28038b1 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-google-cloud-platform.md @@ -0,0 +1,135 @@ +--- +title: Deploy CockroachDB on Google Cloud Platform GCE +summary: Learn how to deploy CockroachDB on Google Cloud Platform's Compute Engine. +toc: true +toc_not_nested: true +ssh-link: https://cloud.google.com/compute/docs/instances/connecting-to-instance +--- + +
+ + +
+ +This page shows you how to manually deploy a secure multi-node CockroachDB cluster on Google Cloud Platform's Compute Engine (GCE), using Google's TCP Proxy Load Balancing service to distribute client traffic. + +If you are only testing CockroachDB, or you are not concerned with protecting network communication with TLS encryption, you can use an insecure cluster instead. Select **Insecure** above for instructions. + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/secure-requirements.md %} + +- This article covers the use of Linux instances with GCE. You may wish to review the instructions for [connecting to Windows instances](https://cloud.google.com/compute/docs/instances/connecting-to-instance#windows). + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/secure-recommendations.md %} + +## Step 1. Configure your network + +CockroachDB requires TCP communication on two ports: + +- `26257` for inter-node communication (i.e., working as a cluster) +- `8080` for exposing your Admin UI + +To expose your Admin UI and allow traffic from the TCP proxy load balancer and health checker to your instances, [create firewall rules](https://cloud.google.com/compute/docs/vpc/firewalls) for your project. When creating firewall rules, we recommend using Google Cloud Platform's **tag** feature to apply the rule only to instances with the same tag. + +#### Admin UI + + Field | Recommended Value +-------|------------------- + Name | **cockroachadmin** + Source filter | IP ranges + Source IP ranges | Your local network's IP ranges + Allowed protocols... | **tcp:8080** + Target tags | **cockroachdb** + +#### Application data + +Applications will not connect directly to your CockroachDB nodes. Instead, they'll connect to GCE's TCP Proxy Load Balancing service, which automatically routes traffic to the instances that are closest to the user. Because this service is implemented at the edge of the Google Cloud, you'll need to create a firewall rule to allow traffic from the load balancer and health checker to your instances. This is covered in [Step 4](#step-4-set-up-load-balancing). + +## Step 2. Create instances + +[Create an instance](https://cloud.google.com/compute/docs/instances/create-start-instance) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate instance for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use `n1-standard` or `n1-highcpu` [predefined VMs](https://cloud.google.com/compute/pricing#predefined_machine_types), or [custom VMs](https://cloud.google.com/compute/pricing#custommachinetypepricing), with [Local SSDs](https://cloud.google.com/compute/docs/disks/#localssds) or [SSD persistent disks](https://cloud.google.com/compute/docs/disks/#pdspecs). For example, Cockroach Labs has used `n1-standard-16` (16 vCPUs and 60 GB of RAM per VM, local SSD) for internal testing. + +- **Do not** use `f1` or `g1` [shared-core machines](https://cloud.google.com/compute/docs/machine-types#sharedcore), which limit the load on a single core. + +- If you used a tag for your firewall rules, when you create the instance, click **Management, security, disks, networking, sole tenancy**. Under the **Networking** tab, in the **Network tags** field, enter **cockroachdb**. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +GCE offers fully-managed [TCP Proxy Load Balancing](https://cloud.google.com/load-balancing/docs/tcp/). This service lets you use a single IP address for all users around the world, automatically routing traffic to the instances that are closest to the user. + +{{site.data.alerts.callout_danger}} +When using TCP Proxy Load Balancing, you cannot use firewall rules to control access to the load balancer. If you need such control, consider using [Network TCP Load Balancing](https://cloud.google.com/compute/docs/load-balancing/network/) instead, but note that it cannot be used across regions. You might also consider using the HAProxy load balancer (see the [On-Premises](deploy-cockroachdb-on-premises.html) tutorial for guidance). +{{site.data.alerts.end}} + +To use GCE's TCP Proxy Load Balancing service: + +1. For each zone in which you're running an instance, [create a distinct instance group](https://cloud.google.com/compute/docs/instance-groups/creating-groups-of-unmanaged-instances). + - To ensure that the load balancer knows where to direct traffic, specify a port name mapping, with `tcp26257` as the **Port name** and `26257` as the **Port number**. +2. [Add the relevant instances to each instance group](https://cloud.google.com/compute/docs/instance-groups/creating-groups-of-unmanaged-instances#addinstances). +3. [Configure TCP Proxy Load Balancing](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#configure_load_balancer). + - During backend configuration, create a health check, setting the **Protocol** to `HTTPS`, the **Port** to `8080`, and the **Request path** to path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + - If you want to maintain long-lived SQL connections that may be idle for more than tens of seconds, increase the backend timeout setting accordingly. + - During frontend configuration, reserve a static IP address and note the IP address and the port you select. You'll use this address and port for all client connections. +4. [Create a firewall rule](https://cloud.google.com/load-balancing/docs/tcp/setting-up-tcp#config-hc-firewall) to allow traffic from the load balancer and health checker to your instances. This is necessary because TCP Proxy Load Balancing is implemented at the edge of the Google Cloud. + - Be sure to set **Source IP ranges** to `130.211.0.0/22` and `35.191.0.0/16` and set **Target tags** to `cockroachdb` (not to the value specified in the linked instructions). + +## Step 5. Generate certificates + +{% include {{ page.version.version }}/prod-deployment/secure-generate-certificates.md %} + +## Step 6. Start nodes + +{{site.data.alerts.callout_info}} +By default, inter-node communication uses the internal IP addresses of your GCE instances. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/prod-deployment/secure-start-nodes.md %} + +## Step 7. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-initialize-cluster.md %} + +## Step 8. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-test-cluster.md %} + +## Step 9. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/secure-test-load-balancing.md %} + +## Step 10. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 11. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-scale-cluster.md %} + +## Step 12. Use the database + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-microsoft-azure-insecure.md b/v20.2/deploy-cockroachdb-on-microsoft-azure-insecure.md new file mode 100644 index 00000000000..7352a22dff3 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-microsoft-azure-insecure.md @@ -0,0 +1,144 @@ +--- +title: Deploy CockroachDB on Microsoft Azure (Insecure) +summary: Learn how to deploy CockroachDB on Microsoft Azure. +toc: true +toc_not_nested: true +ssh-link: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys +--- + + + +This page shows you how to manually deploy an insecure multi-node CockroachDB cluster on Microsoft Azure, using Azure's managed load balancing service to distribute client traffic. + +{{site.data.alerts.callout_danger}}If you plan to use CockroachDB in production, we strongly recommend using a secure cluster instead. Select Secure above for instructions.{{site.data.alerts.end}} + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/insecure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/insecure-recommendations.md %} + +## Step 1. Configure your network + +CockroachDB requires TCP communication on two ports: + +- **26257** (`tcp:26257`) for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- **8080** (`tcp:8080`) for exposing your Admin UI + +To enable this in Azure, you must create a Resource Group, Virtual Network, and Network Security Group. + +1. [Create a Resource Group](https://azure.microsoft.com/en-us/updates/create-empty-resource-groups/). + +2. [Create a Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-create-vnet-arm-pportal) that uses your **Resource Group**. + +3. [Create a Network Security Group](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-create-nsg-arm-pportal) that uses your **Resource Group**, and then add the following **inbound** rules to it: + - **Admin UI support**: + + Field | Recommended Value + -------|------------------- + Name | **cockroachadmin** + Source | **IP Addresses** + Source IP addresses/CIDR ranges | Your local network’s IP ranges + Source port ranges | * + Destination | **Any** + Destination port range | **8080** + Protocol | **TCP** + Action | **Allow** + Priority | Any value > 1000 + - **Application support**: + + {{site.data.alerts.callout_success}}If your application is also hosted on the same Azure Virtual Network, you will not need to create a firewall rule for your application to communicate with your load balancer.{{site.data.alerts.end}} + + Field | Recommended Value + -------|------------------- + Name | **cockroachapp** + Source | **IP Addresses** + Source IP addresses/CIDR ranges | Your local network’s IP ranges + Source port ranges | * + Destination | **Any** + Destination port range | **26257** + Protocol | **TCP** + Action | **Allow** + Priority | Any value > 1000 + + +## Step 2. Create VMs + +[Create Linux VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-linux-quick-create-portal) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate VM for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use compute-optimized [F-series](https://docs.microsoft.com/en-us/azure/virtual-machines/fsv2-series) VMs with [Premium Storage](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/premium-storage) or local SSD storage with a Linux filesystem such as `ext4` (not the Windows `ntfs` filesystem). For example, Cockroach Labs has used `Standard_F16s_v2` VMs (16 vCPUs and 32 GiB of RAM per VM) for internal testing. + + - If you choose local SSD storage, on reboot, the VM can come back with the `ntfs` filesystem. Be sure your automation monitors for this and reformats the disk to the Linux filesystem you chose initially. + +- **Do not** use ["burstable" B-series](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/b-series-burstable) VMs, which limit the load on a single core. Also, Cockroach Labs has experienced data corruption issues on A-series VMs and irregular disk performance on D-series VMs, so we recommend avoiding those as well. + +- When creating the VMs, make sure to select the **Resource Group**, **Virtual Network**, and **Network Security Group** you created. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +Microsoft Azure offers fully-managed load balancing to distribute traffic between instances. + +1. [Add Azure load balancing](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-overview). Be sure to: + - Set forwarding rules to route TCP traffic from the load balancer's port **26257** to port **26257** on the nodes. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + +2. Note the provisioned **IP Address** for the load balancer. You'll use this later to test load balancing and to connect your application to the cluster. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of Azure's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 5. Start nodes + +{% include {{ page.version.version }}/prod-deployment/insecure-start-nodes.md %} + +## Step 6. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-initialize-cluster.md %} + +## Step 7. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-test-cluster.md %} + +## Step 8. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/insecure-test-load-balancing.md %} + +## Step 9. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 10. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-scale-cluster.md %} + +## Step 11. Use the cluster + +Now that your deployment is working, you can: + +1. [Implement your data model](sql-statements.html). +2. [Create users](create-user.html) and [grant them privileges](grant.html). +3. [Connect your application](install-client-drivers.html). Be sure to connect your application to the Azure load balancer, not to a CockroachDB node. + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-microsoft-azure.md b/v20.2/deploy-cockroachdb-on-microsoft-azure.md new file mode 100644 index 00000000000..56d79fe77fe --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-microsoft-azure.md @@ -0,0 +1,141 @@ +--- +title: Deploy CockroachDB on Microsoft Azure +summary: Learn how to deploy CockroachDB on Microsoft Azure. +toc: true +toc_not_nested: true +ssh-link: https://docs.microsoft.com/en-us/azure/virtual-machines/linux/mac-create-ssh-keys +--- + +
+ + +
+ +This page shows you how to manually deploy a secure multi-node CockroachDB cluster on Microsoft Azure, using Azure's managed load balancing service to distribute client traffic. + +If you are only testing CockroachDB, or you are not concerned with protecting network communication with TLS encryption, you can use an insecure cluster instead. Select **Insecure** above for instructions. + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/secure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/secure-recommendations.md %} + +## Step 1. Configure your network + +CockroachDB requires TCP communication on two ports: + +- **26257** (`tcp:26257`) for inter-node communication (i.e., working as a cluster), for applications to connect to the load balancer, and for routing from the load balancer to nodes +- **8080** (`tcp:8080`) for exposing your Admin UI + +To enable this in Azure, you must create a Resource Group, Virtual Network, and Network Security Group. + +1. [Create a Resource Group](https://azure.microsoft.com/en-us/updates/create-empty-resource-groups/). +2. [Create a Virtual Network](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-create-vnet-arm-pportal) that uses your **Resource Group**. +3. [Create a Network Security Group](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-create-nsg-arm-pportal) that uses your **Resource Group**, and then add the following **inbound** rules to it: + - **Admin UI support**: + + Field | Recommended Value + -------|------------------- + Name | **cockroachadmin** + Source | **IP Addresses** + Source IP addresses/CIDR ranges | Your local network’s IP ranges + Source port ranges | * + Destination | **Any** + Destination port range | **8080** + Protocol | **TCP** + Action | **Allow** + Priority | Any value > 1000 + - **Application support**: + + {{site.data.alerts.callout_success}}If your application is also hosted on the same Azure Virtual Network, you will not need to create a firewall rule for your application to communicate with your load balancer.{{site.data.alerts.end}} + + Field | Recommended Value + -------|------------------- + Name | **cockroachapp** + Source | **IP Addresses** + Source IP addresses/CIDR ranges | Your local network’s IP ranges + Source port ranges | * + Destination | **Any** + Destination port range | **26257** + Protocol | **TCP** + Action | **Allow** + Priority | Any value > 1000 + +## Step 2. Create VMs + +[Create Linux VMs](https://docs.microsoft.com/en-us/azure/virtual-machines/virtual-machines-linux-quick-create-portal) for each node you plan to have in your cluster. If you plan to run a sample workload against the cluster, create a separate VM for that workload. + +- Run at least 3 nodes to [ensure survivability](recommended-production-settings.html#topology). + +- Use compute-optimized [F-series](https://docs.microsoft.com/en-us/azure/virtual-machines/fsv2-series) VMs with [Premium Storage](https://docs.microsoft.com/en-us/azure/virtual-machines/windows/premium-storage) or local SSD storage with a Linux filesystem such as `ext4` (not the Windows `ntfs` filesystem). For example, Cockroach Labs has used `Standard_F16s_v2` VMs (16 vCPUs and 32 GiB of RAM per VM) for internal testing. + + - If you choose local SSD storage, on reboot, the VM can come back with the `ntfs` filesystem. Be sure your automation monitors for this and reformats the disk to the Linux filesystem you chose initially. + +- **Do not** use ["burstable" B-series](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/b-series-burstable) VMs, which limit the load on a single core. Also, Cockroach Labs has experienced data corruption issues on A-series VMs and irregular disk performance on D-series VMs, so we recommend avoiding those as well. + +- When creating the VMs, make sure to select the **Resource Group**, **Virtual Network**, and **Network Security Group** you created. + +For more details, see [Hardware Recommendations](recommended-production-settings.html#hardware) and [Cluster Topology](recommended-production-settings.html#topology). + +## Step 3. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 4. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + +Microsoft Azure offers fully-managed load balancing to distribute traffic between instances. + +1. [Add Azure load balancing](https://docs.microsoft.com/en-us/azure/load-balancer/load-balancer-overview). Be sure to: + - Set forwarding rules to route TCP traffic from the load balancer's port **26257** to port **26257** on the nodes. + - Configure health checks to use HTTP port **8080** and path `/health?ready=1`. This [health endpoint](monitoring-and-alerting.html#health-ready-1) ensures that load balancers do not direct traffic to nodes that are live but not ready to receive requests. + +2. Note the provisioned **IP Address** for the load balancer. You'll use this later to test load balancing and to connect your application to the cluster. + +{{site.data.alerts.callout_info}}If you would prefer to use HAProxy instead of Azure's managed load balancing, see the On-Premises tutorial for guidance.{{site.data.alerts.end}} + +## Step 5. Generate certificates + +{% include {{ page.version.version }}/prod-deployment/secure-generate-certificates.md %} + +## Step 6. Start nodes + +{% include {{ page.version.version }}/prod-deployment/secure-start-nodes.md %} + +## Step 7. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-initialize-cluster.md %} + +## Step 8. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-test-cluster.md %} + +## Step 9. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/secure-test-load-balancing.md %} + +## Step 10. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 11. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-scale-cluster.md %} + +## Step 12. Use the database + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-premises-insecure.md b/v20.2/deploy-cockroachdb-on-premises-insecure.md new file mode 100644 index 00000000000..1990e896a30 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-premises-insecure.md @@ -0,0 +1,120 @@ +--- +title: Deploy CockroachDB On-Premises (Insecure) +summary: Learn how to manually deploy an insecure, multi-node CockroachDB cluster on multiple machines. +toc: true +ssh-link: https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2 +redirect_from: manual-deployment-insecure.html + +--- + + + +This tutorial shows you how to manually deploy an insecure multi-node CockroachDB cluster on multiple machines, using [HAProxy](http://www.haproxy.org/) load balancers to distribute client traffic. + +{{site.data.alerts.callout_danger}}If you plan to use CockroachDB in production, we strongly recommend using a secure cluster instead. Select Secure above for instructions.{{site.data.alerts.end}} + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/insecure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/insecure-recommendations.md %} + +## Step 1. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 2. Start nodes + +{% include {{ page.version.version }}/prod-deployment/insecure-start-nodes.md %} + +## Step 3. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-initialize-cluster.md %} + +## Step 4. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-test-cluster.md %} + +## Step 5. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + {{site.data.alerts.callout_success}}With a single load balancer, client connections are resilient to node failure, but the load balancer itself is a point of failure. It's therefore best to make load balancing resilient as well by using multiple load balancing instances, with a mechanism like floating IPs or DNS to select load balancers for clients.{{site.data.alerts.end}} + +[HAProxy](http://www.haproxy.org/) is one of the most popular open-source TCP load balancers, and CockroachDB includes a built-in command for generating a configuration file that is preset to work with your running cluster, so we feature that tool here. + +1. SSH to the machine where you want to run HAProxy. + +2. Install HAProxy: + + {% include copy-clipboard.html %} + ~~~ shell + $ apt-get install haproxy + ~~~ + +3. Download the [CockroachDB archive](https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz) for Linux, and extract the binary: + + {% include copy-clipboard.html %} + ~~~ shell + $ wget -qO- https://binaries.cockroachdb.com/cockroach-{{ page.release_info.version }}.linux-amd64.tgz \ + | tar xvz + ~~~ + +4. Copy the binary into the `PATH`: + + {% include copy-clipboard.html %} + ~~~ shell + $ cp -i cockroach-{{ page.release_info.version }}.linux-amd64/cockroach /usr/local/bin/ + ~~~ + + If you get a permissions error, prefix the command with `sudo`. + +5. Run the [`cockroach gen haproxy`](cockroach-gen.html) command, specifying the address of any CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach gen haproxy --insecure \ + --host=
\ + --port=26257 + ~~~ + + {% include {{ page.version.version }}/misc/haproxy.md %} + +6. Start HAProxy, with the `-f` flag pointing to the `haproxy.cfg` file: + + {% include copy-clipboard.html %} + ~~~ shell + $ haproxy -f haproxy.cfg + ~~~ + +7. Repeat these steps for each additional instance of HAProxy you want to run. + +## Step 6. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/insecure-test-load-balancing.md %} + +## Step 7. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 8. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/insecure-scale-cluster.md %} + +## Step 9. Use the cluster + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/deploy-cockroachdb-on-premises.md b/v20.2/deploy-cockroachdb-on-premises.md new file mode 100644 index 00000000000..738cd544fe7 --- /dev/null +++ b/v20.2/deploy-cockroachdb-on-premises.md @@ -0,0 +1,113 @@ +--- +title: Deploy CockroachDB On-Premises +summary: Learn how to manually deploy a secure, multi-node CockroachDB cluster on multiple machines. +toc: true +ssh-link: https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2 + +--- + + + +This tutorial shows you how to manually deploy a secure multi-node CockroachDB cluster on multiple machines, using [HAProxy](http://www.haproxy.org/) load balancers to distribute client traffic. + +If you are only testing CockroachDB, or you are not concerned with protecting network communication with TLS encryption, you can use an insecure cluster instead. Select **Insecure** above for instructions. + +## Before you begin + +### Requirements + +{% include {{ page.version.version }}/prod-deployment/secure-requirements.md %} + +### Recommendations + +{% include {{ page.version.version }}/prod-deployment/secure-recommendations.md %} + +## Step 1. Synchronize clocks + +{% include {{ page.version.version }}/prod-deployment/synchronize-clocks.md %} + +## Step 2. Generate certificates + +{% include {{ page.version.version }}/prod-deployment/secure-generate-certificates.md %} + +## Step 3. Start nodes + +{% include {{ page.version.version }}/prod-deployment/secure-start-nodes.md %} + +## Step 4. Initialize the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-initialize-cluster.md %} + +## Step 5. Test the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-test-cluster.md %} + +## Step 6. Set up load balancing + +Each CockroachDB node is an equally suitable SQL gateway to your cluster, but to ensure client performance and reliability, it's important to use load balancing: + +- **Performance:** Load balancers spread client traffic across nodes. This prevents any one node from being overwhelmed by requests and improves overall cluster performance (queries per second). + +- **Reliability:** Load balancers decouple client health from the health of a single CockroachDB node. In cases where a node fails, the load balancer redirects client traffic to available nodes. + {{site.data.alerts.callout_success}}With a single load balancer, client connections are resilient to node failure, but the load balancer itself is a point of failure. It's therefore best to make load balancing resilient as well by using multiple load balancing instances, with a mechanism like floating IPs or DNS to select load balancers for clients.{{site.data.alerts.end}} + +[HAProxy](http://www.haproxy.org/) is one of the most popular open-source TCP load balancers, and CockroachDB includes a built-in command for generating a configuration file that is preset to work with your running cluster, so we feature that tool here. + +1. On your local machine, run the [`cockroach gen haproxy`](cockroach-gen.html) command with the `--host` flag set to the address of any node and security flags pointing to the CA cert and the client cert and key: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach gen haproxy \ + --certs-dir=certs \ + --host=
+ ~~~ + + {% include {{ page.version.version }}/misc/haproxy.md %} + +2. Upload the `haproxy.cfg` file to the machine where you want to run HAProxy: + + {% include copy-clipboard.html %} + ~~~ shell + $ scp haproxy.cfg @:~/ + ~~~ + +3. SSH to the machine where you want to run HAProxy. + +4. Install HAProxy: + + {% include copy-clipboard.html %} + ~~~ shell + $ apt-get install haproxy + ~~~ + +5. Start HAProxy, with the `-f` flag pointing to the `haproxy.cfg` file: + + {% include copy-clipboard.html %} + ~~~ shell + $ haproxy -f haproxy.cfg + ~~~ + +6. Repeat these steps for each additional instance of HAProxy you want to run. + +## Step 7. Run a sample workload + +{% include {{ page.version.version }}/prod-deployment/secure-test-load-balancing.md %} + +## Step 8. Monitor the cluster + +{% include {{ page.version.version }}/prod-deployment/monitor-cluster.md %} + +## Step 9. Scale the cluster + +{% include {{ page.version.version }}/prod-deployment/secure-scale-cluster.md %} + +## Step 10. Use the cluster + +{% include {{ page.version.version }}/prod-deployment/use-cluster.md %} + +## See also + +{% include {{ page.version.version }}/prod-deployment/prod-see-also.md %} diff --git a/v20.2/detailed-sql-support.md b/v20.2/detailed-sql-support.md new file mode 100644 index 00000000000..73a55801978 --- /dev/null +++ b/v20.2/detailed-sql-support.md @@ -0,0 +1,688 @@ +--- +title: Detailed SQL Standard Comparison +summary: Details of CockroachDB's conformance to the SQL standard and which common extensions it supports. +--- + +This page lists which SQL standard features are supported, partially-supported, and unsupported by CockroachDB. + +To understand the extent to which we support the standard SQL features, use the table below. + +- **Feature ID** is the SQL Standard feature identification number. +- **Description** provides details about the feature. +- **CockroachDB Support** indicates whether the feature is supported, unsupported, or partially-supported by CockroachDB. + +## Features + +| Feature ID | Description | CockroachDB Support | +| ------ | ------ | ------ | +| B011 | Embedded Ada | No | +| B012 | Embedded C | No | +| B013 | Embedded COBOL | No | +| B014 | Embedded Fortran | No | +| B015 | Embedded MUMPS | No | +| B016 | Embedded Pascal | No | +| B017 | Embedded PL/I | No | +| B021 | Direct SQL | No | +| B031 | Basic dynamic SQL | No | +| B032 | Extended dynamic SQL | No | +| B032-01 | <describe input statement> | No | +| B033 | Untyped SQL-invoked function arguments | No | +| B034 | Dynamic specification of cursor attributes | No | +| B035 | Non-extended descriptor names | No | +| B041 | Extensions to embedded SQL exception declarations | No | +| B051 | Enhanced execution rights | No | +| B111 | Module language Ada | No | +| B112 | Module language C | No | +| B113 | Module language COBOL | No | +| B114 | Module language Fortran | No | +| B115 | Module language MUMPS | No | +| B116 | Module language Pascal | No | +| B117 | Module language PL/I | No | +| B121 | Routine language Ada | No | +| B122 | Routine language C | No | +| B123 | Routine language COBOL | No | +| B124 | Routine language Fortran | No | +| B125 | Routine language MUMPS | No | +| B126 | Routine language Pascal | No | +| B127 | Routine language PL/I | No | +| B128 | Routine language SQL | No | +| B211 | Module language Ada: VARCHAR and NUMERIC support | No | +| B221 | Routine language Ada: VARCHAR and NUMERIC support | No | +| E011 | Numeric data types | Yes | +| E011-01 | INTEGER and SMALLINT data types | Yes | +| E011-02 | REAL, DOUBLE PRECISION, and FLOAT data types | Yes | +| E011-03 | DECIMAL and NUMERIC data types | Yes | +| E011-04 | Arithmetic operators | Yes | +| E011-05 | Numeric comparison | Yes | +| E011-06 | Implicit casting among the numeric data types | Yes | +| E021 | Character data types | Partial | +| E021-01 | CHARACTER data type | Yes | +| E021-02 | CHARACTER VARYING data type | Partial | +| E021-03 | Character literals | Yes | +| E021-04 | CHARACTER_LENGTH function | No | +| E021-05 | OCTET_LENGTH function | Yes | +| E021-06 | SUBSTRING function | Yes | +| E021-07 | Character concatenation | Yes | +| E021-08 | UPPER and LOWER functions | Yes | +| E021-09 | TRIM function | Yes | +| E021-10 | Implicit casting among the character string types | Yes | +| E021-11 | POSITION function | Yes | +| E021-12 | Character comparison | Yes | +| E031 | Identifiers | Yes | +| E031-01 | Delimited identifiers | Yes | +| E031-02 | Lower case identifiers | Yes | +| E031-03 | Trailing underscore | Yes | +| E051 | Basic query specification | Yes | +| E051-01 | SELECT DISTINCT | Yes | +| E051-02 | GROUP BY clause | Yes | +| E051-04 | GROUP BY can contain columns not in <select list> | Yes | +| E051-05 | Select list items can be renamed | Yes | +| E051-06 | HAVING clause | Yes | +| E051-07 | Qualified * in select list | Yes | +| E051-08 | Correlation names in the FROM clause | Yes | +| E051-09 | Rename columns in the FROM clause | Yes | +| E061 | Basic predicates and search conditions | Partial | +| E061-01 | Comparison predicate | Yes | +| E061-02 | BETWEEN predicate | Yes | +| E061-03 | IN predicate with list of values | Yes | +| E061-04 | LIKE predicate | Yes | +| E061-05 | LIKE predicate ESCAPE clause | No | +| E061-06 | NULL predicate | Yes | +| E061-07 | Quantified comparison predicate | Yes | +| E061-08 | EXISTS predicate | Yes | +| E061-09 | Subqueries in comparison predicate | Yes | +| E061-11 | Subqueries in IN predicate | Yes | +| E061-12 | Subqueries in quantified comparison predicate | Yes | +| E061-13 | Correlated subqueries | Partial | +| E061-14 | Search condition | Yes | +| E071 | Basic query expressions | Partial | +| E071-01 | UNION DISTINCT table operator | Yes | +| E071-02 | UNION ALL table operator | Yes | +| E071-03 | EXCEPT DISTINCT table operator | Yes | +| E071-05 | Columns combined via table operators need not have exactly the same data type | No | +| E071-06 | Table operators in subqueries | Yes | +| E081 | Basic Privileges | Partial | +| E081-01 | SELECT privilege | Yes | +| E081-02 | DELETE privilege | Yes | +| E081-03 | INSERT privilege at the table level | Yes | +| E081-04 | UPDATE privilege at the table level | Yes | +| E081-05 | UPDATE privilege at the column level | No | +| E081-06 | REFERENCES privilege at the table level | No | +| E081-07 | REFERENCES privilege at the column level | No | +| E081-08 | WITH GRANT OPTION | No | +| E081-09 | USAGE privilege | Yes | +| E081-10 | EXECUTE privilege | No | +| E091 | Set functions | Yes | +| E091-01 | AVG | Yes | +| E091-02 | COUNT | Yes | +| E091-03 | MAX | Yes | +| E091-04 | MIN | Yes | +| E091-05 | SUM | Yes | +| E091-06 | ALL quantifier | Yes | +| E091-07 | DISTINCT quantifier | Yes | +| E101 | Basic data manipulation | Yes | +| E101-01 | INSERT statement | Yes | +| E101-03 | Searched UPDATE statement | Yes | +| E101-04 | Searched DELETE statement | Yes | +| E111 | Single row SELECT statement | Yes | +| E121 | Basic cursor support | No | +| E121-01 | DECLARE CURSOR | No | +| E121-02 | ORDER BY columns need not be in select list | No | +| E121-03 | Value expressions in ORDER BY clause | No | +| E121-04 | OPEN statement | No | +| E121-06 | Positioned UPDATE statement | No | +| E121-07 | Positioned DELETE statement | No | +| E121-08 | CLOSE statement | No | +| E121-10 | FETCH statement implicit NEXT | No | +| E121-17 | WITH HOLD cursors | No | +| E131 | Null value support (nulls in lieu of values) | Yes | +| E141 | Basic integrity constraints | Yes | +| E141-01 | NOT NULL constraints | Yes | +| E141-02 | UNIQUE constraints of NOT NULL columns | Yes | +| E141-03 | PRIMARY KEY constraints | Yes | +| E141-04 | Basic FOREIGN KEY constraint with the NO ACTION default for both referential delete action and referential update action | Yes | +| E141-06 | CHECK constraints | Yes | +| E141-07 | Column defaults | Yes | +| E141-08 | NOT NULL inferred on PRIMARY KEY | Yes | +| E141-10 | Names in a foreign key can be specified in any order | Yes | +| E151 | Transaction support | Yes | +| E151-01 | COMMIT statement | Yes | +| E151-02 | ROLLBACK statement | Yes | +| E152 | Basic SET TRANSACTION statement | Partial | +| E152-01 | SET TRANSACTION statement: ISOLATION LEVEL SERIALIZABLE clause | Yes | +| E152-02 | SET TRANSACTION statement: READ ONLY and READ WRITE clauses | No | +| E153 | Updatable queries with subqueries | No | +| E161 | SQL comments using leading double minus | Yes | +| E171 | SQLSTATE support | Partial | +| E182 | Module language | No | +| F021 | Basic information schema | Yes | +| F021-01 | COLUMNS view | Yes | +| F021-02 | TABLES view | Yes | +| F021-03 | VIEWS view | Yes | +| F021-04 | TABLE_CONSTRAINTS view | Yes | +| F021-05 | REFERENTIAL_CONSTRAINTS view | Yes | +| F021-06 | CHECK_CONSTRAINTS view | Yes | +| F031 | Basic schema manipulation | Yes | +| F031-01 | CREATE TABLE statement to create persistent base tables | Yes | +| F031-02 | CREATE VIEW statement | Yes | +| F031-03 | GRANT statement | Yes | +| F031-04 | ALTER TABLE statement: ADD COLUMN clause | Yes | +| F031-13 | DROP TABLE statement: RESTRICT clause | Yes | +| F031-16 | DROP VIEW statement: RESTRICT clause | Yes | +| F031-19 | REVOKE statement: RESTRICT clause | Yes | +| F032 | CASCADE drop behavior | Yes | +| F033 | ALTER TABLE statement: DROP COLUMN clause | Yes | +| F034 | Extended REVOKE statement | Partial | +| F034-01 | REVOKE statement performed by other than the owner of a schema object | Yes | +| F034-02 | REVOKE statement: GRANT OPTION FOR clause | No | +| F034-03 | REVOKE statement to revoke a privilege that the grantee has WITH GRANT OPTION | No | +| F041 | Basic joined table | Yes | +| F041-01 | Inner join (but not necessarily the INNER keyword) | Yes | +| F041-02 | INNER keyword | Yes | +| F041-03 | LEFT OUTER JOIN | Yes | +| F041-04 | RIGHT OUTER JOIN | Yes | +| F041-05 | Outer joins can be nested | Yes | +| F041-07 | The inner table in a left or right outer join can also be used in an inner join | Yes | +| F041-08 | All comparison operators are supported (rather than just =) | Yes | +| F051 | Basic date and time | Partial | +| F051-01 | DATE data type (including support of DATE literal) | Yes | +| F051-02 | TIME data type (including support of TIME literal) with fractional seconds precision of at least 0 | Yes | +| F051-03 | TIMESTAMP data type (including support of TIMESTAMP literal) with fractional seconds precision of at least 0 and 6 | Yes | +| F051-04 | Comparison predicate on DATE, TIME, and TIMESTAMP data types | Yes | +| F051-05 | Explicit CAST between datetime types and character string types | Yes | +| F051-06 | CURRENT_DATE | Yes | +| F051-07 | LOCALTIME | No | +| F051-08 | LOCALTIMESTAMP | No | +| F052 | Intervals and datetime arithmetic | Yes | +| F053 | OVERLAPS predicate | No | +| F054 | TIMESTAMP in DATE type precedence list | No | +| F081 | UNION and EXCEPT in views | Yes | +| F111 | Isolation levels other than SERIALIZABLE | No | +| F111-01 | READ UNCOMMITTED isolation level | No | +| F111-02 | READ COMMITTED isolation level | No | +| F111-03 | REPEATABLE READ isolation level | No | +| F121 | Basic diagnostics management | No | +| F121-01 | GET DIAGNOSTICS statement | No | +| F121-02 | SET TRANSACTION statement: DIAGNOSTICS SIZE clause | No | +| F122 | Enhanced diagnostics management | No | +| F123 | All diagnostics | No | +| F131 | Grouped operations | Yes | +| F131-01 | WHERE, GROUP BY, and HAVING clauses supported in queries with grouped views | Yes | +| F131-02 | Multiple tables supported in queries with grouped views | Yes | +| F131-03 | Set functions supported in queries with grouped views | Yes | +| F131-04 | Subqueries with GROUP BY and HAVING clauses and grouped views | Yes | +| F131-05 | Single row SELECT with GROUP BY and HAVING clauses and grouped views | Yes | +| F171 | Multiple schemas per user | No | +| F181 | Multiple module support | No | +| F191 | Referential delete actions | No | +| F200 | TRUNCATE TABLE statement | Yes | +| F201 | CAST function | Yes | +| F202 | TRUNCATE TABLE: identity column restart option | No | +| F221 | Explicit defaults | Yes | +| F222 | INSERT statement: DEFAULT VALUES clause | Yes | +| F231 | Privilege tables | Yes | +| F231-01 | TABLE_PRIVILEGES view | Yes | +| F231-02 | COLUMN_PRIVILEGES view | No | +| F231-03 | USAGE_PRIVILEGES view | No | +| F251 | Domain support | No | +| F261 | CASE expression | Yes | +| F261-01 | Simple CASE | Yes | +| F261-02 | Searched CASE | Yes | +| F261-03 | NULLIF | Yes | +| F261-04 | COALESCE | Yes | +| F262 | Extended CASE expression | No | +| F263 | Comma-separated predicates in simple CASE expression | No | +| F271 | Compound character literals | No | +| F281 | LIKE enhancements | Yes | +| F291 | UNIQUE predicate | Yes | +| F301 | CORRESPONDING in query expressions | No | +| F302 | INTERSECT table operator | Yes | +| F302-01 | INTERSECT DISTINCT table operator | Yes | +| F302-02 | INTERSECT ALL table operator | Yes | +| F304 | EXCEPT ALL table operator | Yes | +| F311 | Schema definition statement | No | +| F311-01 | CREATE SCHEMA | No | +| F311-02 | CREATE TABLE for persistent base tables | Yes | +| F311-03 | CREATE VIEW | Yes | +| F311-04 | CREATE VIEW: WITH CHECK OPTION | No | +| F311-05 | GRANT statement | Yes | +| F312 | MERGE statement | No | +| F313 | Enhanced MERGE statement | No | +| F314 | MERGE statement with DELETE branch | No | +| F321 | User authorization | Partial | +| F341 | Usage tables | No | +| F361 | Subprogram support | No | +| F381 | Extended schema manipulation | Partial | +| F381-01 | ALTER TABLE statement: ALTER COLUMN clause | Partial | +| F381-02 | ALTER TABLE statement: ADD CONSTRAINT clause | Yes | +| F381-03 | ALTER TABLE statement: DROP CONSTRAINT clause | Yes | +| F382 | Alter column data type | No | +| F383 | Set column not null clause | No | +| F384 | Drop identity property clause | No | +| F385 | Drop column generation expression clause | No | +| F386 | Set identity column generation clause | No | +| F391 | Long identifiers | Yes | +| F392 | Unicode escapes in identifiers | Yes | +| F393 | Unicode escapes in literals | Yes | +| F394 | Optional normal form specification | No | +| F401 | Extended joined table | Yes | +| F401-01 | NATURAL JOIN | Yes | +| F401-02 | FULL OUTER JOIN | Yes | +| F401-04 | CROSS JOIN | Yes | +| F402 | Named column joins for LOBs, arrays, and multisets | Partial | +| F403 | Partitioned joined tables | No | +| F411 | Time zone specification | Yes | +| F421 | National character | No | +| F431 | Read-only scrollable cursors | No | +| F431-01 | FETCH with explicit NEXT | No | +| F431-02 | FETCH FIRST | No | +| F431-03 | FETCH LAST | No | +| F431-04 | FETCH PRIOR | No | +| F431-05 | FETCH ABSOLUTE | No | +| F431-06 | FETCH RELATIVE | No | +| F441 | Extended set function support | Yes | +| F442 | Mixed column references in set functions | Yes | +| F451 | Character set definition | No | +| F461 | Named character sets | No | +| F471 | Scalar subquery values | Yes | +| F481 | Expanded NULL predicate | Yes | +| F491 | Constraint management | Yes | +| F492 | Optional table constraint enforcement | Partial | +| F501 | Features and conformance views | No | +| F501-01 | SQL_FEATURES view | No | +| F501-02 | SQL_SIZING view | No | +| F501-03 | SQL_LANGUAGES view | No | +| F502 | Enhanced documentation tables | No | +| F502-01 | SQL_SIZING_PROFILES view | No | +| F502-02 | SQL_IMPLEMENTATION_INFO view | No | +| F502-03 | SQL_PACKAGES view | No | +| F521 | Assertions | No | +| F531 | Temporary tables | No | +| F555 | Enhanced seconds precision | No | +| F561 | Full value expressions | Yes | +| F571 | Truth value tests | Yes | +| F591 | Derived tables | Yes | +| F611 | Indicator data types | No | +| F641 | Row and table constructors | Yes | +| F651 | Catalog name qualifiers | Partial | +| F661 | Simple tables | Yes | +| F671 | Subqueries in CHECK | No | +| F672 | Retrospective check constraints | Yes | +| F690 | Collation support | Yes | +| F692 | Extended collation support | Yes | +| F693 | SQL-session and client module collations | Yes | +| F695 | Translation support | No | +| F696 | Additional translation documentation | No | +| F701 | Referential update actions | No | +| F711 | ALTER domain | No | +| F721 | Deferrable constraints | No | +| F731 | INSERT column privileges | No | +| F741 | Referential MATCH types | No | +| F751 | View CHECK enhancements | No | +| F761 | Session management | No | +| F762 | CURRENT_CATALOG | No | +| F763 | CURRENT_SCHEMA | Partial | +| F771 | Connection management | No | +| F781 | Self-referencing operations | No | +| F791 | Insensitive cursors | No | +| F801 | Full set function | No | +| F812 | Basic flagging | No | +| F813 | Extended flagging | No | +| F821 | Local table references | No | +| F831 | Full cursor update | No | +| F831-01 | Updatable scrollable cursors | No | +| F831-02 | Updatable ordered cursors | No | +| F841 | LIKE_REGEX predicate | Partial | +| F842 | OCCURRENCES_REGEX function | No | +| F843 | POSITION_REGEX function | No | +| F844 | SUBSTRING_REGEX function | Partial | +| F845 | TRANSLATE_REGEX function | Partial | +| F846 | Octet support in regular expression operators | No | +| F847 | Nonconstant regular expressions | Yes | +| F850 | Top-level <order by clause> in <query expression> | Yes | +| F851 | <order by clause> in subqueries | Yes | +| F852 | Top-level <order by clause> in views | Yes | +| F855 | Nested <order by clause> in <query expression> | Yes | +| F856 | Nested <fetch first clause> in <query expression> | Yes | +| F857 | Top-level <fetch first clause> in <query expression> | Yes | +| F858 | <fetch first clause> in subqueries | Yes | +| F859 | Top-level <fetch first clause> in views | Yes | +| F860 | <fetch first row count> in <fetch first clause> | Yes | +| F861 | Top-level <result offset clause> in <query expression> | Yes | +| F862 | <result offset clause> in subqueries | Yes | +| F863 | Nested <result offset clause> in <query expression> | Yes | +| F864 | Top-level <result offset clause> in views | Yes | +| F865 | <offset row count> in <result offset clause> | Yes | +| F866 | FETCH FIRST clause: PERCENT option | No | +| F867 | FETCH FIRST clause: WITH TIES option | No | +| M001 | Datalinks | No | +| M002 | Datalinks via SQL/CLI | No | +| M003 | Datalinks via Embedded SQL | No | +| M004 | Foreign data support | No | +| M005 | Foreign schema support | No | +| M006 | GetSQLString routine | No | +| M007 | TransmitRequest | No | +| M009 | GetOpts and GetStatistics routines | No | +| M010 | Foreign data wrapper support | No | +| M011 | Datalinks via Ada | No | +| M012 | Datalinks via C | No | +| M013 | Datalinks via COBOL | No | +| M014 | Datalinks via Fortran | No | +| M015 | Datalinks via M | No | +| M016 | Datalinks via Pascal | No | +| M017 | Datalinks via PL/I | No | +| M018 | Foreign data wrapper interface routines in Ada | No | +| M019 | Foreign data wrapper interface routines in C | No | +| M020 | Foreign data wrapper interface routines in COBOL | No | +| M021 | Foreign data wrapper interface routines in Fortran | No | +| M022 | Foreign data wrapper interface routines in MUMPS | No | +| M023 | Foreign data wrapper interface routines in Pascal | No | +| M024 | Foreign data wrapper interface routines in PL/I | No | +| M030 | SQL-server foreign data support | No | +| M031 | Foreign data wrapper general routines | No | +| S011 | Distinct data types | No | +| S011-01 | USER_DEFINED_TYPES view | No | +| S023 | Basic structured types | No | +| S024 | Enhanced structured types | No | +| S025 | Final structured types | No | +| S026 | Self-referencing structured types | No | +| S027 | Create method by specific method name | No | +| S028 | Permutable UDT options list | No | +| S041 | Basic reference types | No | +| S043 | Enhanced reference types | No | +| S051 | Create table of type | No | +| S071 | SQL paths in function and type name resolution | No | +| S081 | Subtables | No | +| S091 | Basic array support | Yes | +| S091-01 | Arrays of built-in data types | Yes | +| S091-02 | Arrays of distinct types | No | +| S091-03 | Array expressions | Yes | +| S092 | Arrays of user-defined types | No | +| S094 | Arrays of reference types | No | +| S095 | Array constructors by query | Yes | +| S096 | Optional array bounds | Yes | +| S097 | Array element assignment | No | +| S098 | ARRAY_AGG | Yes | +| S111 | ONLY in query expressions | Yes | +| S151 | Type predicate | Yes | +| S161 | Subtype treatment | No | +| S162 | Subtype treatment for references | No | +| S201 | SQL-invoked routines on arrays | Yes | +| S201-01 | Array parameters | Yes | +| S201-02 | Array as result type of functions | Yes | +| S202 | SQL-invoked routines on multisets | No | +| S211 | User-defined cast functions | No | +| S231 | Structured type locators | No | +| S232 | Array locators | No | +| S233 | Multiset locators | No | +| S241 | Transform functions | No | +| S242 | Alter transform statement | No | +| S251 | User-defined orderings | No | +| S261 | Specific type method | No | +| S271 | Basic multiset support | No | +| S272 | Multisets of user-defined types | No | +| S274 | Multisets of reference types | No | +| S275 | Advanced multiset support | No | +| S281 | Nested collection types | No | +| S291 | Unique constraint on entire row | No | +| S301 | Enhanced UNNEST | No | +| S401 | Distinct types based on array types | No | +| S402 | Distinct types based on distinct types | No | +| S403 | ARRAY_MAX_CARDINALITY | No | +| S404 | TRIM_ARRAY | No | +| T011 | Timestamp in Information Schema | No | +| T021 | BINARY and VARBINARY data types | No | +| T022 | Advanced support for BINARY and VARBINARY data types | No | +| T023 | Compound binary literal | No | +| T024 | Spaces in binary literals | No | +| T031 | BOOLEAN data type | Yes | +| T041 | Basic LOB data type support | No | +| T041-01 | BLOB data type | Partial | +| T041-02 | CLOB data type | No | +| T041-03 | POSITION, LENGTH, LOWER, TRIM, UPPER, and SUBSTRING functions for LOB data types | Partial | +| T041-04 | Concatenation of LOB data types | Yes | +| T041-05 | LOB locator: non-holdable | No | +| T042 | Extended LOB data type support | Partial | +| T043 | Multiplier T | No | +| T044 | Multiplier P | No | +| T051 | Row types | Yes | +| T052 | MAX and MIN for row types | Yes | +| T053 | Explicit aliases for all-fields reference | No | +| T061 | UCS support | No | +| T071 | BIGINT data type | Yes | +| T101 | Enhanced nullability determination | No | +| T111 | Updatable joins, unions, and columns | No | +| T121 | WITH (excluding RECURSIVE) in query expression | Partial | +| T122 | WITH (excluding RECURSIVE) in subquery | Partial | +| T131 | Recursive query | No | +| T132 | Recursive query in subquery | No | +| T141 | SIMILAR predicate | Yes | +| T151 | DISTINCT predicate | Yes | +| T152 | DISTINCT predicate with negation | Yes | +| T171 | LIKE clause in table definition | No | +| T172 | AS subquery clause in table definition | Yes | +| T173 | Extended LIKE clause in table definition | No | +| T174 | Identity columns | No | +| T175 | Generated columns | No | +| T176 | Sequence generator support | Yes | +| T177 | Sequence generator support: simple restart option | No | +| T178 | Identity columns: simple restart option | No | +| T180 | System-versioned tables | No | +| T181 | Application-time period tables | No | +| T191 | Referential action RESTRICT | Yes | +| T201 | Comparable data types for referential constraints | No | +| T211 | Basic trigger capability | No | +| T211-01 | Triggers activated on UPDATE, INSERT, or DELETE of one base table | No | +| T211-02 | BEFORE triggers | No | +| T211-03 | AFTER triggers | No | +| T211-04 | FOR EACH ROW triggers | No | +| T211-05 | Ability to specify a search condition that must be true before the trigger is invoked | No | +| T211-06 | Support for run-time rules for the interaction of triggers and constraints | No | +| T211-07 | TRIGGER privilege | No | +| T211-08 | Multiple triggers for the same event are executed in the order in which they were created in the catalog | No | +| T212 | Enhanced trigger capability | No | +| T213 | INSTEAD OF triggers | No | +| T231 | Sensitive cursors | No | +| T241 | START TRANSACTION statement | Yes | +| T251 | SET TRANSACTION statement: LOCAL option | No | +| T261 | Chained transactions | No | +| T271 | Savepoints | Yes | +| T272 | Enhanced savepoint management | No | +| T281 | SELECT privilege with column granularity | No | +| T285 | Enhanced derived column names | No | +| T301 | Functional dependencies | No | +| T312 | OVERLAY function | Yes | +| T321 | Basic SQL-invoked routines | No | +| T321-01 | User-defined functions with no overloading | No | +| T321-02 | User-defined stored procedures with no overloading | No | +| T321-03 | Function invocation | No | +| T321-04 | CALL statement | No | +| T321-05 | RETURN statement | No | +| T321-06 | ROUTINES view | No | +| T321-07 | PARAMETERS view | No | +| T322 | Declared data type attributes | No | +| T323 | Explicit security for external routines | No | +| T324 | Explicit security for SQL routines | No | +| T325 | Qualified SQL parameter references | No | +| T326 | Table functions | Partial | +| T331 | Basic roles | Partial | +| T332 | Extended roles | No | +| T341 | Overloading of SQL-invoked functions and procedures | Yes | +| T351 | Bracketed SQL comments (/*...*/ comments) | Yes | +| T431 | Extended grouping capabilities | No | +| T432 | Nested and concatenated GROUPING SETS | No | +| T433 | Multiargument GROUPING function | No | +| T434 | GROUP BY DISTINCT | Yes | +| T441 | ABS and MOD functions | Yes | +| T461 | Symmetric BETWEEN predicate | Yes | +| T471 | Result sets return value | No | +| T472 | DESCRIBE CURSOR | No | +| T491 | LATERAL derived table | Yes | +| T495 | Combined data change and retrieval | Partial | +| T501 | Enhanced EXISTS predicate | Yes | +| T502 | Period predicates | No | +| T511 | Transaction counts | No | +| T521 | Named arguments in CALL statement | No | +| T522 | Default values for IN parameters of SQL-invoked procedures | No | +| T551 | Optional key words for default syntax | Yes | +| T561 | Holdable locators | No | +| T571 | Array-returning external SQL-invoked functions | No | +| T572 | Multiset-returning external SQL-invoked functions | No | +| T581 | Regular expression substring function | Yes | +| T591 | UNIQUE constraints of possibly null columns | Yes | +| T601 | Local cursor references | No | +| T611 | Elementary OLAP operations | No | +| T612 | Advanced OLAP operations | No | +| T613 | Sampling | No | +| T614 | NTILE function | Yes | +| T615 | LEAD and LAG functions | Yes | +| T616 | Null treatment option for LEAD and LAG functions | No | +| T617 | FIRST_VALUE and LAST_VALUE function | Yes | +| T618 | NTH_VALUE function | Partial | +| T619 | Nested window functions | No | +| T620 | WINDOW clause: GROUPS option | No | +| T621 | Enhanced numeric functions | No | +| T631 | IN predicate with one list element | Yes | +| T641 | Multiple column assignment | Partial | +| T651 | SQL-schema statements in SQL routines | No | +| T652 | SQL-dynamic statements in SQL routines | No | +| T653 | SQL-schema statements in external routines | No | +| T654 | SQL-dynamic statements in external routines | No | +| T655 | Cyclically dependent routines | No | +| X010 | XML type | No | +| X011 | Arrays of XML type | No | +| X012 | Multisets of XML type | No | +| X013 | Distinct types of XML type | No | +| X014 | Attributes of XML type | No | +| X015 | Fields of XML type | No | +| X016 | Persistent XML values | No | +| X020 | XMLConcat | No | +| X025 | XMLCast | No | +| X030 | XMLDocument | No | +| X031 | XMLElement | No | +| X032 | XMLForest | No | +| X034 | XMLAgg | No | +| X035 | XMLAgg: ORDER BY option | No | +| X036 | XMLComment | No | +| X037 | XMLPI | No | +| X038 | XMLText | No | +| X040 | Basic table mapping | No | +| X041 | Basic table mapping: nulls absent | No | +| X042 | Basic table mapping: null as nil | No | +| X043 | Basic table mapping: table as forest | No | +| X044 | Basic table mapping: table as element | No | +| X045 | Basic table mapping: with target namespace | No | +| X046 | Basic table mapping: data mapping | No | +| X047 | Basic table mapping: metadata mapping | No | +| X048 | Basic table mapping: base64 encoding of binary strings | No | +| X049 | Basic table mapping: hex encoding of binary strings | No | +| X050 | Advanced table mapping | No | +| X051 | Advanced table mapping: nulls absent | No | +| X052 | Advanced table mapping: null as nil | No | +| X053 | Advanced table mapping: table as forest | No | +| X054 | Advanced table mapping: table as element | No | +| X055 | Advanced table mapping: with target namespace | No | +| X056 | Advanced table mapping: data mapping | No | +| X057 | Advanced table mapping: metadata mapping | No | +| X058 | Advanced table mapping: base64 encoding of binary strings | No | +| X059 | Advanced table mapping: hex encoding of binary strings | No | +| X060 | XMLParse: character string input and CONTENT option | No | +| X061 | XMLParse: character string input and DOCUMENT option | No | +| X065 | XMLParse: BLOB input and CONTENT option | No | +| X066 | XMLParse: BLOB input and DOCUMENT option | No | +| X068 | XMLSerialize: BOM | No | +| X069 | XMLSerialize: INDENT | No | +| X070 | XMLSerialize: character string serialization and CONTENT option | No | +| X071 | XMLSerialize: character string serialization and DOCUMENT option | No | +| X072 | XMLSerialize: character string serialization | No | +| X073 | XMLSerialize: BLOB serialization and CONTENT option | No | +| X074 | XMLSerialize: BLOB serialization and DOCUMENT option | No | +| X075 | XMLSerialize: BLOB serialization | No | +| X076 | XMLSerialize: VERSION | No | +| X077 | XMLSerialize: explicit ENCODING option | No | +| X078 | XMLSerialize: explicit XML declaration | No | +| X080 | Namespaces in XML publishing | No | +| X081 | Query-level XML namespace declarations | No | +| X082 | XML namespace declarations in DML | No | +| X083 | XML namespace declarations in DDL | No | +| X084 | XML namespace declarations in compound statements | No | +| X085 | Predefined namespace prefixes | No | +| X086 | XML namespace declarations in XMLTable | No | +| X090 | XML document predicate | No | +| X091 | XML content predicate | No | +| X096 | XMLExists | No | +| X100 | Host language support for XML: CONTENT option | No | +| X101 | Host language support for XML: DOCUMENT option | No | +| X110 | Host language support for XML: VARCHAR mapping | No | +| X111 | Host language support for XML: CLOB mapping | No | +| X112 | Host language support for XML: BLOB mapping | No | +| X113 | Host language support for XML: STRIP WHITESPACE option | No | +| X114 | Host language support for XML: PRESERVE WHITESPACE option | No | +| X120 | XML parameters in SQL routines | No | +| X121 | XML parameters in external routines | No | +| X131 | Query-level XMLBINARY clause | No | +| X132 | XMLBINARY clause in DML | No | +| X133 | XMLBINARY clause in DDL | No | +| X134 | XMLBINARY clause in compound statements | No | +| X135 | XMLBINARY clause in subqueries | No | +| X141 | IS VALID predicate: data-driven case | No | +| X142 | IS VALID predicate: ACCORDING TO clause | No | +| X143 | IS VALID predicate: ELEMENT clause | No | +| X144 | IS VALID predicate: schema location | No | +| X145 | IS VALID predicate outside check constraints | No | +| X151 | IS VALID predicate with DOCUMENT option | No | +| X152 | IS VALID predicate with CONTENT option | No | +| X153 | IS VALID predicate with SEQUENCE option | No | +| X155 | IS VALID predicate: NAMESPACE without ELEMENT clause | No | +| X157 | IS VALID predicate: NO NAMESPACE with ELEMENT clause | No | +| X160 | Basic Information Schema for registered XML Schemas | No | +| X161 | Advanced Information Schema for registered XML Schemas | No | +| X170 | XML null handling options | No | +| X171 | NIL ON NO CONTENT option | No | +| X181 | XML(DOCUMENT(UNTYPED)) type | No | +| X182 | XML(DOCUMENT(ANY)) type | No | +| X190 | XML(SEQUENCE) type | No | +| X191 | XML(DOCUMENT(XMLSCHEMA)) type | No | +| X192 | XML(CONTENT(XMLSCHEMA)) type | No | +| X200 | XMLQuery | No | +| X201 | XMLQuery: RETURNING CONTENT | No | +| X202 | XMLQuery: RETURNING SEQUENCE | No | +| X203 | XMLQuery: passing a context item | No | +| X204 | XMLQuery: initializing an XQuery variable | No | +| X205 | XMLQuery: EMPTY ON EMPTY option | No | +| X206 | XMLQuery: NULL ON EMPTY option | No | +| X211 | XML 1.1 support | No | +| X221 | XML passing mechanism BY VALUE | No | +| X222 | XML passing mechanism BY REF | No | +| X231 | XML(CONTENT(UNTYPED)) type | No | +| X232 | XML(CONTENT(ANY)) type | No | +| X241 | RETURNING CONTENT in XML publishing | No | +| X242 | RETURNING SEQUENCE in XML publishing | No | +| X251 | Persistent XML values of XML(DOCUMENT(UNTYPED)) type | No | +| X252 | Persistent XML values of XML(DOCUMENT(ANY)) type | No | +| X253 | Persistent XML values of XML(CONTENT(UNTYPED)) type | No | +| X254 | Persistent XML values of XML(CONTENT(ANY)) type | No | +| X255 | Persistent XML values of XML(SEQUENCE) type | No | +| X256 | Persistent XML values of XML(DOCUMENT(XMLSCHEMA)) type | No | +| X257 | Persistent XML values of XML(CONTENT(XMLSCHEMA)) type | No | +| X260 | XML type: ELEMENT clause | No | +| X261 | XML type: NAMESPACE without ELEMENT clause | No | +| X263 | XML type: NO NAMESPACE with ELEMENT clause | No | +| X264 | XML type: schema location | No | +| X271 | XMLValidate: data-driven case | No | +| X272 | XMLValidate: ACCORDING TO clause | No | +| X273 | XMLValidate: ELEMENT clause | No | +| X274 | XMLValidate: schema location | No | +| X281 | XMLValidate with DOCUMENT option | No | +| X282 | XMLValidate with CONTENT option | No | +| X283 | XMLValidate with SEQUENCE option | No | +| X284 | XMLValidate: NAMESPACE without ELEMENT clause | No | +| X286 | XMLValidate: NO NAMESPACE with ELEMENT clause | No | +| X300 | XMLTable | No | +| X301 | XMLTable: derived column list option | No | +| X302 | XMLTable: ordinality column option | No | +| X303 | XMLTable: column default option | No | +| X304 | XMLTable: passing a context item | No | +| X305 | XMLTable: initializing an XQuery variable | No | +| X400 | Name and identifier mapping | No | +| X410 | Alter column data type: XML type | No | diff --git a/v20.2/developer-guide-overview.md b/v20.2/developer-guide-overview.md new file mode 100644 index 00000000000..c00ed534622 --- /dev/null +++ b/v20.2/developer-guide-overview.md @@ -0,0 +1,31 @@ +--- +title: Developer Guide Overview +summary: An overview of common tasks that come up when you build an application using CockroachDB +toc: true +--- + +This guide shows you how to do common tasks that come up when you build an application using CockroachDB. + +## Common Tasks + +- [Install a Postgres Client](install-client-drivers.html) +- [Connect to the Database](connect-to-the-database.html) +- [Insert Data](insert-data.html) +- [Query Data](query-data.html) +- [Update Data](update-data.html) +- [Delete Data](delete-data.html) +- [Run Multi-Statement Transactions](run-multi-statement-transactions.html) +- [Error Handling and Troubleshooting](error-handling-and-troubleshooting.html) +- [Make Queries Fast](make-queries-fast.html) + +## Sample Apps and Tutorials + +For simple but complete sample apps and tutorials, see [Hello World](hello-world-example-apps.html). + +## See also + +- [Migrate to CockroachDB](migration-overview.html) +- [Connection parameters](connection-parameters.html) +- [Selection queries](selection-queries.html) +- [Joins](joins.html) +- [Transactions](transactions.html) diff --git a/v20.2/diagnostics-reporting.md b/v20.2/diagnostics-reporting.md new file mode 100644 index 00000000000..c415cf19896 --- /dev/null +++ b/v20.2/diagnostics-reporting.md @@ -0,0 +1,66 @@ +--- +title: Diagnostics Reporting +summary: Learn about the diagnostic details that get shared with CockroachDB and how to opt out of sharing. +toc: true +--- + +By default, the Admin UI and each node of a CockroachDB cluster share anonymous usage details with Cockroach Labs. These details, which are completely scrubbed of identifiable information, greatly help us understand and improve how the system behaves in real-world scenarios. + +This page summarizes the details that get shared, how to view the details yourself, and how to opt out of sharing. + +{{site.data.alerts.callout_success}} +For insights into your cluster's performance and health, use the built-in [Admin UI](admin-ui-overview.html) or a third-party monitoring tool like [Prometheus](monitor-cockroachdb-with-prometheus.html). +{{site.data.alerts.end}} + +## What gets shared + +When diagnostics reporting is on, each node of a CockroachDB cluster shares anonymized details on an hourly basis, including (but not limited to): + +- Deployment and configuration characteristics, such as size of hardware, [cluster settings](cluster-settings.html) that have been altered from defaults, number of [replication zones](configure-replication-zones.html) configured, etc. +- Usage and cluster health details, such as crashes, unexpected errors, attempts to use unsupported features, types of queries run and their execution characteristics as well as types of schemas used, etc. + +To view the full diagnostics details that a node reports to Cockroach Labs, use the `http://:/_status/diagnostics/local` JSON endpoint. + +{{site.data.alerts.callout_info}} +In all cases, names and other string values are scrubbed and replaced with underscores. Also, the details that get shared may change over time, but as that happens, we will announce the changes in release notes. +{{site.data.alerts.end}} + +## Opt out of diagnostics reporting + +### At cluster initialization + +To make sure that absolutely no diagnostic details are shared, you can set the environment variable `COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING=true` before starting the first node of the cluster. Note that this works only when set before starting the first node of the cluster. Once the cluster is running, you need to use the `SET CLUSTER SETTING` method described below. + +### After cluster initialization + +To stop sending diagnostic details to Cockroach Labs once a cluster is running, [use the built-in SQL client](cockroach-sql.html) to execute the following [`SET CLUSTER SETTING`](set-cluster-setting.html) statement, which switches the `diagnostics.reporting.enabled` [cluster setting](cluster-settings.html) to `false`: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING diagnostics.reporting.enabled = false; +~~~ + +This change will not be instantaneous, as it must be propagated to other nodes in the cluster. + +## Check the state of diagnostics reporting + +To check the state of diagnostics reporting, [use the built-in SQL client](cockroach-sql.html) to execute the following [`SHOW CLUSTER SETTING`](show-cluster-setting.html) statement: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CLUSTER SETTING diagnostics.reporting.enabled; +~~~ + +~~~ + diagnostics.reporting.enabled ++-------------------------------+ + false +(1 row) +~~~ + +If the setting is `false`, diagnostics reporting is off; if the setting is `true`, diagnostics reporting is on. + +## See also + +- [Cluster Settings](cluster-settings.html) +- [Start a Node](cockroach-start.html) diff --git a/v20.2/drop-column.md b/v20.2/drop-column.md new file mode 100644 index 00000000000..4129215c632 --- /dev/null +++ b/v20.2/drop-column.md @@ -0,0 +1,93 @@ +--- +title: DROP COLUMN +summary: Use the ALTER COLUMN statement to remove columns from tables. +toc: true +--- + +The `DROP COLUMN` [statement](sql-statements.html) is part of `ALTER TABLE` and removes columns from a table. + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_column.html %}
+ +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + + Parameter | Description +-----------|------------- + `table_name` | The name of the table with the column you want to drop. + `name` | The name of the column you want to drop.

When a column with a `CHECK` constraint is dropped, the `CHECK` constraint is also dropped. + `CASCADE` | Drop the column even if objects (such as [views](views.html)) depend on it; drop the dependent objects, as well.

`CASCADE` does not list objects it drops, so should be used cautiously. However, `CASCADE` will not drop dependent indexes; you must use [`DROP INDEX`](drop-index.html).

`CASCADE` will drop a column with a foreign key constraint if it is the only column in the reference. + `RESTRICT` | *(Default)* Do not drop the column if any objects (such as [views](views.html)) depend on it. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Drop columns + +If you no longer want a column in a table, you can drop it. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE orders DROP COLUMN billing_zip; +~~~ + +### Prevent dropping columns with dependent objects (`RESTRICT`) + +If the column has dependent objects, such as [views](views.html), CockroachDB will not drop the column by default; however, if you want to be sure of the behavior you can include the `RESTRICT` clause. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE orders DROP COLUMN customer RESTRICT; +~~~ +~~~ +pq: cannot drop column "customer" because view "customer_view" depends on it +~~~ + +### Drop column and dependent objects (`CASCADE`) + +If you want to drop the column and all of its dependent options, include the `CASCADE` clause. + +{{site.data.alerts.callout_danger}}CASCADE does not list objects it drops, so should be used cautiously.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE customer_view; +~~~ + +~~~ ++---------------+----------------------------------------------------------------+ +| table_name | create_statement | ++---------------+----------------------------------------------------------------+ +| customer_view | CREATE VIEW customer_view AS SELECT customer FROM store.orders | ++---------------+----------------------------------------------------------------+ +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE orders DROP COLUMN customer CASCADE; +~~~ + +{% include copy-clipboard.html %} +~~~ +> SHOW CREATE customer_view; +~~~ + +~~~ +pq: view "customer_view" does not exist +~~~ + +## See also + +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`DROP INDEX`](drop-index.html) +- [`ALTER TABLE`](alter-table.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/drop-constraint.md b/v20.2/drop-constraint.md new file mode 100644 index 00000000000..0399ce4e4ad --- /dev/null +++ b/v20.2/drop-constraint.md @@ -0,0 +1,155 @@ +--- +title: DROP CONSTRAINT +summary: Use the ALTER CONSTRAINT statement to remove constraints from columns. +toc: true +--- + +The `DROP CONSTRAINT` [statement](sql-statements.html) is part of [`ALTER TABLE`](alter-table.html) and removes [`CHECK`](check.html) and [`FOREIGN KEY`](foreign-key.html) constraints from columns. + + [`PRIMARY KEY`](primary-key.html) constraints can be dropped with `DROP CONSTRAINT` if an [`ADD CONSTRAINT`](add-constraint.html) statement follows the `DROP CONSTRAINT` statement in the same transaction. + +{{site.data.alerts.callout_success}} +When you change a primary key with [`ALTER TABLE ... ALTER PRIMARY KEY`](alter-primary-key.html), the old primary key index becomes a secondary index. If you do not want the old primary key to become a secondary index, use `DROP CONSTRAINT`/[`ADD CONSTRAINT`](add-constraint.html) to change the primary key. +{{site.data.alerts.end}} + +{{site.data.alerts.callout_info}} +For information about removing other constraints, see [Constraints: Remove Constraints](constraints.html#remove-constraints). +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_constraint.html %}
+ +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on the table. + +## Parameters + + Parameter | Description +-----------|------------- + `table_name` | The name of the table with the constraint you want to drop. + `name` | The name of the constraint you want to drop. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Drop a foreign key constraint + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CONSTRAINTS FROM vehicles; +~~~ + +~~~ + table_name | constraint_name | constraint_type | details | validated +-------------+-------------------+-----------------+---------------------------------------------------------+------------ + vehicles | fk_city_ref_users | FOREIGN KEY | FOREIGN KEY (city, owner_id) REFERENCES users(city, id) | true + vehicles | primary | PRIMARY KEY | PRIMARY KEY (city ASC, id ASC) | true +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE vehicles DROP CONSTRAINT fk_city_ref_users; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CONSTRAINTS FROM vehicles; +~~~ + +~~~ + table_name | constraint_name | constraint_type | details | validated +-------------+-----------------+-----------------+--------------------------------+------------ + vehicles | primary | PRIMARY KEY | PRIMARY KEY (city ASC, id ASC) | true +(1 row) +~~~ + +### Drop and add a primary key constraint + +When you change a primary key with [`ALTER TABLE ... ALTER PRIMARY KEY`](alter-primary-key.html), the old primary key index becomes a secondary index. If you do not want the old primary key to become a secondary index when changing a primary key, you can use `DROP CONSTRAINT`/[`ADD CONSTRAINT`](add-constraint.html) instead. + +Suppose that you want to add `name` to the composite primary key of the `users` table. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+-------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) +(1 row) +~~~ + +First, add a [`NOT NULL`](not-null.html) constraint to the `name` column with [`ALTER COLUMN`](alter-column.html). + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE users ALTER COLUMN name SET NOT NULL; +~~~ + +Then, in the same transaction, `DROP` the old `"primary"` constraint and [`ADD`](add-constraint.html) the new one: + +{% include copy-clipboard.html %} +~~~ sql +> BEGIN; +> ALTER TABLE users DROP CONSTRAINT "primary"; +> ALTER TABLE users ADD CONSTRAINT "primary" PRIMARY KEY (city, name, id); +> COMMIT; +~~~ + +~~~ +NOTICE: primary key changes are finalized asynchronously; further schema changes on this table may be restricted until the job completes +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE users; +~~~ + +~~~ + table_name | create_statement +-------------+--------------------------------------------------------------------- + users | CREATE TABLE users ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | name VARCHAR NOT NULL, + | address VARCHAR NULL, + | credit_card VARCHAR NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, name ASC, id ASC), + | FAMILY "primary" (id, city, name, address, credit_card) + | ) +(1 row) +~~~ + +Using [`ALTER PRIMARY KEY`](alter-primary-key.html) would have created a `UNIQUE` secondary index called `users_city_id_key`. Instead, there is just one index for the primary key constraint. + +## See also + +- [`ADD CONSTRAINT`](add-constraint.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) +- [`RENAME CONSTRAINT`](rename-constraint.html) +- [`VALIDATE CONSTRAINT`](validate-constraint.html) +- [`DROP COLUMN`](drop-column.html) +- [`DROP INDEX`](drop-index.html) +- [`ALTER TABLE`](alter-table.html) +- [`SHOW JOBS`](show-jobs.html) +- ['ALTER PRIMARY KEY'](alter-primary-key.html) diff --git a/v20.2/drop-database.md b/v20.2/drop-database.md new file mode 100644 index 00000000000..bd6839fe2da --- /dev/null +++ b/v20.2/drop-database.md @@ -0,0 +1,105 @@ +--- +title: DROP DATABASE +summary: The DROP DATABASE statement removes a database and all its objects from a CockroachDB cluster. +toc: true +--- + +The `DROP DATABASE` [statement](sql-statements.html) removes a database and all its objects from a CockroachDB cluster. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `DROP` [privilege](authorization.html#assign-privileges) on the database and on all tables in the database. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_database.html %}
+ +## Parameters + +Parameter | Description +----------|------------ +`IF EXISTS` | Drop the database if it exists; if it does not exist, do not return an error. +`name` | The name of the database you want to drop. You cannot drop a database if it is set as the [current database](sql-name-resolution.html#current-database) or if [`sql_safe_updates = true`](set-vars.html). +`CASCADE` | _(Default)_ Drop all tables and views in the database as well as all objects (such as [constraints](constraints.html) and [views](views.html)) that depend on those tables.

`CASCADE` does not list objects it drops, so should be used cautiously. +`RESTRICT` | Do not drop the database if it contains any [tables](create-table.html) or [views](create-view.html). + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Drop a database and its objects (`CASCADE`) + +For non-interactive sessions (e.g., client applications), `DROP DATABASE` applies the `CASCADE` option by default, which drops all tables and views in the database as well as all objects (such as [constraints](constraints.html) and [views](views.html)) that depend on those tables. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM db2; +~~~ + +~~~ ++------------+ +| table_name | ++------------+ +| t1 | +| v1 | ++------------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP DATABASE db2; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM db2; +~~~ + +~~~ +pq: database "db2" does not exist +~~~ + +For interactive sessions from the [built-in SQL client](cockroach-sql.html), either the `CASCADE` option must be set explicitly or the `--unsafe-updates` flag must be set when starting the shell. + +### Prevent dropping a non-empty database (`RESTRICT`) + +When a database is not empty, the `RESTRICT` option prevents the database from being dropped: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM db2; +~~~ + +~~~ ++------------+ +| table_name | ++------------+ +| t1 | +| v1 | ++------------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP DATABASE db2 RESTRICT; +~~~ + +~~~ +pq: database "db2" is not empty and CASCADE was not specified +~~~ + +## See also + +- [`CREATE DATABASE`](create-database.html) +- [`SHOW DATABASES`](show-databases.html) +- [`RENAME DATABASE`](rename-database.html) +- [`SET DATABASE`](set-vars.html) +- [`SHOW JOBS`](show-jobs.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/drop-index.md b/v20.2/drop-index.md new file mode 100644 index 00000000000..084e32f3e2b --- /dev/null +++ b/v20.2/drop-index.md @@ -0,0 +1,140 @@ +--- +title: DROP INDEX +summary: The DROP INDEX statement removes indexes from tables. +toc: true +--- + +The `DROP INDEX` [statement](sql-statements.html) removes indexes from tables. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_index.html %}
+ +## Required privileges + +The user must have the `CREATE` [privilege](authorization.html#assign-privileges) on each specified table. + +## Parameters + + Parameter | Description +-----------|------------- + `IF EXISTS` | Drop the named indexes if they exist; if they do not exist, do not return an error. + `table_name` | The name of the table with the index you want to drop. Find table names with [`SHOW TABLES`](show-tables.html). + `index_name` | The name of the index you want to drop. Find index names with [`SHOW INDEX`](show-index.html).

You cannot drop a table's `primary` index. + `CASCADE` | Drop all objects (such as [constraints](constraints.html)) that depend on the indexes. `CASCADE` does not list objects it drops, so should be used cautiously.

To drop an index created with [`CREATE UNIQUE INDEX`](create-index.html#unique-indexes), you do not need to use `CASCADE`. + `RESTRICT` | _(Default)_ Do not drop the indexes if any objects (such as [constraints](constraints.html)) depend on them. + `CONCURRENTLY` | Optional, no-op syntax for PostgreSQL compatibility. All indexes are dropped concurrently in CockroachDB. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Remove an index (no dependencies) + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM tl; +~~~ + +~~~ ++------------+-------------+------------+--------------+-------------+-----------+---------+----------+ +| table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit | ++------------+-------------+------------+--------------+-------------+-----------+---------+----------+ +| t1 | primary | false | 1 | id | ASC | false | false | +| t1 | t1_name_idx | true | 1 | name | ASC | false | false | +| t1 | t1_name_idx | true | 2 | id | ASC | false | true | ++------------+-------------+------------+--------------+-------------+-----------+---------+----------+ +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP INDEX t1@t1_name_idx; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM tbl; +~~~ + +~~~ ++------------+------------+------------+--------------+-------------+-----------+---------+----------+ +| table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit | ++------------+------------+------------+--------------+-------------+-----------+---------+----------+ +| t1 | primary | false | 1 | id | ASC | false | false | ++------------+------------+------------+--------------+-------------+-----------+---------+----------+ +(1 row) +~~~ + +### Remove an index and dependent objects with `CASCADE` + +{{site.data.alerts.callout_danger}}CASCADE drops all dependent objects without listing them, which can lead to inadvertent and difficult-to-recover losses. To avoid potential harm, we recommend dropping objects individually in most cases.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SHOW INDEX FROM orders; +~~~ + +~~~ ++------------+---------------------------------------------+------------+--------------+-------------+-----------+---------+----------+ +| table_name | index_name | non_unique | seq_in_index | column_name | direction | storing | implicit | ++------------+---------------------------------------------+------------+--------------+-------------+-----------+---------+----------+ +| orders | primary | false | 1 | id | ASC | false | false | +| orders | orders_auto_index_fk_customer_ref_customers | true | 1 | customer | ASC | false | false | +| orders | orders_auto_index_fk_customer_ref_customers | true | 2 | id | ASC | false | true | ++------------+---------------------------------------------+------------+--------------+-------------+-----------+---------+----------+ +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP INDEX orders_auto_index_fk_customer_ref_customers; +~~~ + +~~~ +pq: index "orders_auto_index_fk_customer_ref_customers" is in use as a foreign key constraint +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CONSTRAINTS FROM orders; +~~~ + +~~~ ++------------+---------------------------+-----------------+--------------------------------------------------+-----------+ +| table_name | constraint_name | constraint_type | details | validated | ++------------+---------------------------+-----------------+--------------------------------------------------+-----------+ +| orders | fk_customer_ref_customers | FOREIGN KEY | FOREIGN KEY (customer) REFERENCES customers (id) | true | +| orders | primary | PRIMARY KEY | PRIMARY KEY (id ASC) | true | ++------------+---------------------------+-----------------+--------------------------------------------------+-----------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP INDEX orders_auto_index_fk_customer_ref_customers CASCADE; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CONSTRAINTS FROM orders; +~~~ + +~~~ ++------------+-----------------+-----------------+----------------------+-----------+ +| table_name | constraint_name | constraint_type | details | validated | ++------------+-----------------+-----------------+----------------------+-----------+ +| orders | primary | PRIMARY KEY | PRIMARY KEY (id ASC) | true | ++------------+-----------------+-----------------+----------------------+-----------+ +(1 row) +~~~ + +## See Also + +- [Indexes](indexes.html) +- [Online Schema Changes](online-schema-changes.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/drop-role.md b/v20.2/drop-role.md new file mode 100644 index 00000000000..45614cb8949 --- /dev/null +++ b/v20.2/drop-role.md @@ -0,0 +1,73 @@ +--- +title: DROP ROLE +summary: The DROP ROLE statement removes one or more SQL roles. +toc: true +--- + +The `DROP ROLE` [statement](sql-statements.html) removes one or more SQL roles. + +{{site.data.alerts.callout_info}} + DROP ROLE is no longer an enterprise feature and is now freely available in the core version of CockroachDB. Also, since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `DROP ROLE` is now an alias for [`DROP USER`](drop-user.html). +{{site.data.alerts.end}} + +## Considerations + +- The `admin` role cannot be dropped, and `root` must always be a member of `admin`. +- A role cannot be dropped if it has privileges. Use [`REVOKE`](revoke.html) to remove privileges. + +## Required privileges + +Roles can only be dropped by super users, i.e., members of the `admin` role. + + To drop other non-admin roles, the role must have the [`CREATEROLE`](create-role.html#allow-the-role-to-create-other-roles) parameter set. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_role.html %}
+ + +## Parameters + + Parameter | Description +------------|-------------- +`name` | The name of the role to remove. To remove multiple roles, use a comma-separate list of roles.

You can use [`SHOW ROLES`](show-roles.html) to find the names of roles. + +## Example + +In this example, first check a role's privileges. Then, revoke the role's privileges and remove the role. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON documents FOR dev_ops; +~~~ +~~~ ++------------+--------+-----------+---------+------------+ +| Database | Schema | Table | User | Privileges | ++------------+--------+-----------+---------+------------+ +| jsonb_test | public | documents | dev_ops | INSERT | ++------------+--------+-----------+---------+------------+ +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> REVOKE INSERT ON documents FROM dev_ops; +~~~ + +{{site.data.alerts.callout_info}}All of a role's privileges must be revoked before the role can be dropped.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> DROP ROLE dev_ops; +~~~ +~~~ +DROP ROLE 1 +~~~ + +## See also + +- [Authorization](authorization.html) +- [`CREATE ROLE`](create-role.html) +- [`SHOW ROLES`](show-roles.html) +- [`GRANT`](grant.html) +- [`SHOW GRANTS`](show-grants.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/drop-sequence.md b/v20.2/drop-sequence.md new file mode 100644 index 00000000000..f055f4844be --- /dev/null +++ b/v20.2/drop-sequence.md @@ -0,0 +1,102 @@ +--- +title: DROP SEQUENCE +summary: +toc: true +--- + +The `DROP SEQUENCE` [statement](sql-statements.html) removes a sequence from a database. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `DROP` [privilege](authorization.html#assign-privileges) on the specified sequence(s). + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_sequence.html %}
+ +## Parameters + + + + Parameter | Description +-----------|------------ +`IF EXISTS` | Drop the sequence only if it exists; if it does not exist, do not return an error. +`sequence_name` | The name of the sequence you want to drop. Find the sequence name with `SHOW CREATE` on the table that uses the sequence. +`RESTRICT` | _(Default)_ Do not drop the sequence if any objects (such as [constraints](constraints.html) and tables) use it. +`CASCADE` | Not yet implemented. Currently, you can only drop a sequence if nothing depends on it. + + + +## Examples + +### Remove a sequence (no dependencies) + +In this example, other objects do not depend on the sequence being dropped. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.sequences; +~~~ +~~~ ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +| sequence_catalog | sequence_schema | sequence_name | data_type | numeric_precision | numeric_precision_radix | numeric_scale | start_value | minimum_value | maximum_value | increment | cycle_option | ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +| def | db_2 | test_4 | INT | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO | +| def | test_db | customer_seq | INT | 64 | 2 | 0 | 101 | 1 | 9223372036854775807 | 2 | NO | +| def | test_db | desc_customer_list | INT | 64 | 2 | 0 | 1000 | -9223372036854775808 | -1 | -2 | NO | +| def | test_db | test_sequence3 | INT | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO | ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +(4 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP SEQUENCE customer_seq; +~~~ +~~~ +DROP SEQUENCE +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.sequences +~~~ +~~~ ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +| sequence_catalog | sequence_schema | sequence_name | data_type | numeric_precision | numeric_precision_radix | numeric_scale | start_value | minimum_value | maximum_value | increment | cycle_option | ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +| def | db_2 | test_4 | INT | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO | +| def | test_db | desc_customer_list | INT | 64 | 2 | 0 | 1000 | -9223372036854775808 | -1 | -2 | NO | +| def | test_db | test_sequence3 | INT | 64 | 2 | 0 | 1 | 1 | 9223372036854775807 | 1 | NO | ++------------------+-----------------+--------------------+-----------+-------------------+-------------------------+---------------+-------------+----------------------+---------------------+-----------+--------------+ +(4 rows) +~~~ + + + + +## See also +- [`CREATE SEQUENCE`](create-sequence.html) +- [`ALTER SEQUENCE`](alter-sequence.html) +- [`RENAME SEQUENCE`](rename-sequence.html) +- [`SHOW SEQUENCES`](show-sequences.html) +- [Functions and Operators](functions-and-operators.html) +- [Other SQL Statements](sql-statements.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/drop-table.md b/v20.2/drop-table.md new file mode 100644 index 00000000000..5e0f485a9ed --- /dev/null +++ b/v20.2/drop-table.md @@ -0,0 +1,204 @@ +--- +title: DROP TABLE +summary: The DROP TABLE statement removes a table and all its indexes from a database. +toc: true +--- + +The `DROP TABLE` [statement](sql-statements.html) removes a table and all its indexes from a database. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `DROP` [privilege](authorization.html#assign-privileges) on the specified table(s). If `CASCADE` is used, the user must have the privileges required to drop each dependent object as well. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_table.html %}
+ +## Parameters + +Parameter | Description +----------|------------ +`IF EXISTS` | Drop the table if it exists; if it does not exist, do not return an error. +`table_name` | A comma-separated list of table names. To find table names, use [`SHOW TABLES`](show-tables.html). +`CASCADE` | Drop all objects (such as [constraints](constraints.html) and [views](views.html)) that depend on the table.

`CASCADE` does not list objects it drops, so should be used cautiously. +`RESTRICT` | _(Default)_ Do not drop the table if any objects (such as [constraints](constraints.html) and [views](views.html)) depend on it. + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +{% include {{page.version.version}}/sql/movr-statements.md %} + +### Remove a table (no dependencies) + +In this example, other objects do not depend on the table being dropped. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM movr; +~~~ + +~~~ + table_name ++----------------------------+ + promo_codes + rides + user_promo_codes + users + vehicle_location_histories + vehicles +(6 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP TABLE promo_codes; +~~~ + +~~~ +DROP TABLE +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM movr; +~~~ + +~~~ + table_name ++----------------------------+ + rides + user_promo_codes + users + vehicle_location_histories + vehicles +(5 rows) +~~~ + +### Remove a table and dependent objects with `CASCADE` + +In this example, a [foreign key](foreign-key.html) from a different table references the table being dropped. Therefore, it's only possible to drop the table while simultaneously dropping the dependent foreign key constraint using `CASCADE`. + +{{site.data.alerts.callout_danger}}CASCADE drops all dependent objects without listing them, which can lead to inadvertent and difficult-to-recover losses. To avoid potential harm, we recommend dropping objects individually in most cases.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM movr; +~~~ + +~~~ + table_name ++----------------------------+ + rides + user_promo_codes + users + vehicle_location_histories + vehicles +(5 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP TABLE users; +~~~ + +~~~ +pq: "users" is referenced by foreign key from table "vehicles" +~~~ + +To see how `users` is referenced from `vehicles`, you can use the [`SHOW CREATE`](show-create.html) statement. `SHOW CREATE` shows how the columns in a table are created, including data types, default values, indexes, and constraints. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE vehicles; +~~~ + +~~~ + table_name | create_statement ++------------+-----------------------------------------------------------------------------------------------+ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city VARCHAR NOT NULL, + | type VARCHAR NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status VARCHAR NULL, + | current_location VARCHAR NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | CONSTRAINT fk_city_ref_users FOREIGN KEY (city, owner_id) REFERENCES users(city, id), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) +(1 row) +~~~ + + +{% include copy-clipboard.html %} +~~~sql +> DROP TABLE users CASCADE; +~~~ + +~~~ +DROP TABLE +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW TABLES FROM movr; +~~~ + +~~~ + table_name ++----------------------------+ + rides + user_promo_codes + vehicle_location_histories + vehicles +(4 rows) +~~~ + +Use a `SHOW CREATE TABLE` statement to verify that the foreign key constraint has been removed from `vehicles`. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CREATE TABLE vehicles; +~~~ + +~~~ + table_name | create_statement ++------------+-----------------------------------------------------------------------------------------------+ + vehicles | CREATE TABLE vehicles ( + | id UUID NOT NULL, + | city STRING NOT NULL, + | type STRING NULL, + | owner_id UUID NULL, + | creation_time TIMESTAMP NULL, + | status STRING NULL, + | current_location STRING NULL, + | ext JSONB NULL, + | CONSTRAINT "primary" PRIMARY KEY (city ASC, id ASC), + | INDEX vehicles_auto_index_fk_city_ref_users (city ASC, owner_id ASC), + | FAMILY "primary" (id, city, type, owner_id, creation_time, status, current_location, ext) + | ) +(1 row) +~~~ + +## See also + +- [`ALTER TABLE`](alter-table.html) +- [`CREATE TABLE`](create-table.html) +- [`INSERT`](insert.html) +- [`RENAME TABLE`](rename-table.html) +- [`SHOW COLUMNS`](show-columns.html) +- [`SHOW TABLES`](show-tables.html) +- [`UPDATE`](update.html) +- [`DELETE`](delete.html) +- [`DROP INDEX`](drop-index.html) +- [`DROP VIEW`](drop-view.html) +- [`SHOW JOBS`](show-jobs.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/drop-user.md b/v20.2/drop-user.md new file mode 100644 index 00000000000..62a12dc4ff5 --- /dev/null +++ b/v20.2/drop-user.md @@ -0,0 +1,67 @@ +--- +title: DROP USER +summary: The DROP USER statement removes one or more SQL users. +toc: true +--- + +The `DROP USER` [statement](sql-statements.html) removes one or more SQL users. + +{{site.data.alerts.callout_info}} + Since the keywords `ROLE` and `USER` can now be used interchangeably in SQL statements for enhanced Postgres compatibility, `DROP USER` is now an alias for [`DROP ROLE`](drop-role.html). +{{site.data.alerts.end}} + +## Required privileges + + To drop other non-admin users, the user must have the [`CREATEROLE`](create-user.html#allow-the-user-to-create-other-users) parameter set. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_user.html %}
+ +## Parameters + + Parameter | Description +-----------|------------- +`user_name` | The username of the user to remove. To remove multiple users, use a comma-separate list of usernames.

You can use [`SHOW USERS`](show-users.html) to find usernames. + +## Example + +All of a user's privileges must be revoked before the user can be dropped. + +In this example, first check a user's privileges. Then, revoke the user's privileges before removing the user. + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON test.customers FOR mroach; +~~~ + +~~~ ++-----------+--------+------------+ +| Table | User | Privileges | ++-----------+--------+------------+ +| customers | mroach | CREATE | +| customers | mroach | INSERT | +| customers | mroach | UPDATE | ++-----------+--------+------------+ +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> REVOKE CREATE,INSERT,UPDATE ON test.customers FROM mroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP USER mroach; +~~~ + +## See also + +- [`CREATE USER`](create-user.html) +- [`ALTER USER`](alter-user.html) +- [`SHOW USERS`](show-users.html) +- [`GRANT`](grant.html) +- [`SHOW GRANTS`](show-grants.html) +- [Create Security Certificates](cockroach-cert.html) +- [Other SQL Statements](sql-statements.html) diff --git a/v20.2/drop-view.md b/v20.2/drop-view.md new file mode 100644 index 00000000000..bd7fb919008 --- /dev/null +++ b/v20.2/drop-view.md @@ -0,0 +1,131 @@ +--- +title: DROP VIEW +summary: The DROP VIEW statement removes a view from a database. +toc: true +--- + +The `DROP VIEW` [statement](sql-statements.html) removes a [view](views.html) from a database. + +{% include {{{ page.version.version }}/misc/schema-change-stmt-note.md %} + +## Required privileges + +The user must have the `DROP` [privilege](authorization.html#assign-privileges) on the specified view(s). If `CASCADE` is used to drop dependent views, the user must have the `DROP` privilege on each dependent view as well. + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/drop_view.html %}
+ +## Parameters + + Parameter | Description +----------|------------- + `IF EXISTS` | Drop the view if it exists; if it does not exist, do not return an error. + `table_name` | A comma-separated list of view names. To find view names, use:

`SELECT * FROM information_schema.tables WHERE table_type = 'VIEW';` + `CASCADE` | Drop other views that depend on the view being dropped.

`CASCADE` does not list views it drops, so should be used cautiously. + `RESTRICT` | _(Default)_ Do not drop the view if other views depend on it. + +## Examples + +### Remove a view (no dependencies) + +In this example, other views do not depend on the view being dropped. + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+--------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+--------------------+------------+---------+ +| def | bank | user_accounts | VIEW | 1 | +| def | bank | user_emails | VIEW | 1 | ++---------------+-------------------+--------------------+------------+---------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP VIEW bank.user_emails; +~~~ + +~~~ +DROP VIEW +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+--------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+--------------------+------------+---------+ +| def | bank | user_accounts | VIEW | 1 | ++---------------+-------------------+--------------------+------------+---------+ +(1 row) +~~~ + +### Remove a view (with dependencies) + +In this example, another view depends on the view being dropped. Therefore, it's only possible to drop the view while simultaneously dropping the dependent view using `CASCADE`. + +{{site.data.alerts.callout_danger}}CASCADE drops all dependent views without listing them, which can lead to inadvertent and difficult-to-recover losses. To avoid potential harm, we recommend dropping objects individually in most cases.{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+--------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+--------------------+------------+---------+ +| def | bank | user_accounts | VIEW | 1 | +| def | bank | user_emails | VIEW | 1 | ++---------------+-------------------+--------------------+------------+---------+ +(2 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> DROP VIEW bank.user_accounts; +~~~ + +~~~ +pq: cannot drop view "user_accounts" because view "user_emails" depends on it +~~~ + +{% include copy-clipboard.html %} +~~~sql +> DROP VIEW bank.user_accounts CASCADE; +~~~ + +~~~ +DROP VIEW +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM information_schema.tables WHERE table_type = 'VIEW'; +~~~ + +~~~ ++---------------+-------------------+--------------------+------------+---------+ +| TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | VERSION | ++---------------+-------------------+--------------------+------------+---------+ +| def | bank | create_test | VIEW | 1 | ++---------------+-------------------+--------------------+------------+---------+ +(1 row) +~~~ + +## See also + +- [Views](views.html) +- [`CREATE VIEW`](create-view.html) +- [`SHOW CREATE`](show-create.html) +- [`ALTER VIEW`](alter-view.html) +- [Online Schema Changes](online-schema-changes.html) diff --git a/v20.2/enable-node-map.md b/v20.2/enable-node-map.md new file mode 100644 index 00000000000..7d5b2f3c315 --- /dev/null +++ b/v20.2/enable-node-map.md @@ -0,0 +1,200 @@ +--- +title: Enable the Node Map +summary: Learn how to enable the node map in the Admin UI. +toc: true +--- + +{{site.data.alerts.callout_info}} +On a secure cluster, this area of the Admin UI can only be accessed by an `admin` user. See [Admin UI access](admin-ui-overview.html#admin-ui-access). +{{site.data.alerts.end}} + +The **Node Map** is useful for: + +- Visualizing the geographic configuration of a multi-region cluster on a world map. +- Viewing real-time cluster metrics. +- Drilling down to individual nodes for monitoring health and performance. + +This page walks you through the process of setting up and enabling the Node Map. + +{{site.data.alerts.callout_info}} +The **Node Map** is an [enterprise-only](enterprise-licensing.html) feature. However, you can [request a trial license](https://www.cockroachlabs.com/get-cockroachdb/) to try it out. +{{site.data.alerts.end}} + +CockroachDB Admin UI + +## Set up and enable the Node Map + +To enable the **Node Map**, you need to start the cluster with the correct [`--locality`](cockroach-start.html#locality) flags and assign the latitude and longitude for each locality. + +{{site.data.alerts.callout_info}} +The Node Map will not be displayed until *all* nodes are started with the correct `--locality` flags and all localities are assigned the corresponding latitude and longitude. +{{site.data.alerts.end}} + +Consider a four-node geo-distributed cluster with the following configuration: + +| Node | Region | Datacenter | +| ------ | ------ | ------ | +| Node1 | us-east-1 | us-east-1a | +| Node2 | us-east-1 | us-east-1b | +| Node3 | us-west-1 | us-west-1a | +| Node4 | eu-west-1 | eu-west-1a | + +### Step 1. Start the nodes with the correct `--locality` flags + +To start a new cluster with the correct `--locality` flags: + +Start Node 1: + +{% include copy-clipboard.html %} +~~~ +$ cockroach start \ +--insecure \ +--locality=region=us-east-1,datacenter=us-east-1a \ +--advertise-addr= \ +--cache=.25 \ +--max-sql-memory=.25 \ +--join=,,, +~~~ + +Start Node 2: + +{% include copy-clipboard.html %} +~~~ +$ cockroach start \ +--insecure \ +--locality=region=us-east-1,datacenter=us-east-1b \ +--advertise-addr= \ +--cache=.25 \ +--max-sql-memory=.25 \ +--join=,,, +~~~ + +Start Node 3: + +{% include copy-clipboard.html %} +~~~ +$ cockroach start \ +--insecure \ +--locality=region=us-west-1,datacenter=us-west-1a \ +--advertise-addr= \ +--cache=.25 \ +--max-sql-memory=.25 \ +--join=,,, +~~~ + +Start Node 4: + +{% include copy-clipboard.html %} +~~~ +$ cockroach start \ +--insecure \ +--locality=region=eu-west-1,datacenter=eu-west-1a \ +--advertise-addr= \ +--cache=.25 \ +--max-sql-memory=.25 \ +--join=,,, +~~~ + +Use the [`cockroach init`](cockroach-init.html) command to perform a one-time initialization of the cluster: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach init --insecure --host=
+~~~ + +[Access the Admin UI](admin-ui-access-and-navigate.html#access-the-admin-ui). The following page is displayed: + +CockroachDB Admin UI + +### Step 2. [Set the enterprise license](enterprise-licensing.html) and refresh the Admin UI + +The following page should be displayed: + +CockroachDB Admin UI + +### Step 3. Set the latitudes and longitudes for the localities + +Launch the built-in SQL client: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure --host=
+~~~ + +Insert the approximate latitude and longitude of each region into the `system.locations` table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO system.locations VALUES + ('region', 'us-east-1', 37.478397, -76.453077), + ('region', 'us-west-1', 38.837522, -120.895824), + ('region', 'eu-west-1', 53.142367, -7.692054); +~~~ + +{{site.data.alerts.callout_info}} +The Node Map will not be displayed until each region is assigned a corresponding latitude and longitude. {{site.data.alerts.end}} + +For the latitudes and longitudes of AWS, Azure, and Google Cloud regions, see [Location Coordinates for Reference](#location-coordinates). + +### Step 4. View the Node Map + +[Open the **Overview page**](admin-ui-access-and-navigate.html) and select **Node Map** from the **View** drop-down menu. The **Node Map** will be displayed: + +CockroachDB Admin UI + +### Step 5. Navigate the Node Map + +Let's say you want to navigate to Node 2, which is in datacenter `us-east-1a` in the `us-east-1` region: + +1. Click on the map component marked as **region=us-east-1** on the Node Map. The datacenter view is displayed. +2. Click on the datacenter component marked as **datacenter=us-east-1a**. The individual node components are displayed. +3. To navigate back to the cluster view, either click on **Cluster** in the breadcrumb trail at the top of the Node Map, or click **Up to region=us-east-1** and then click **Up to Cluster** in the lower left-hand side of the Node Map. + +CockroachDB Admin UI + +## Troubleshoot the Node Map + +### Node Map not displayed + +The **Node Map** will not be displayed until all nodes have localities and are assigned the corresponding latitudes and longitudes. To verify this, navigate to the Localities debug page (`https://
:8080/#/reports/localities`) in the Admin UI. + +The Localities debug page displays the following: + +- Localities configuration that you set up while starting the nodes with the `--locality` flags. +- Nodes corresponding to each locality. +- Latitude and longitude coordinates for each locality/node. + +On the page, ensure that each node is assigned a locality and latitude/longitude coordinates. + +### Node Map not displayed for all locality levels + +The **Node Map** is displayed only for the locality levels that have latitude/longitude coordinates assigned to them: + +- If you assign the latitude/longitude coordinates at the region level, the Node Map shows the regions on the world map. However, when you drill down to the datacenter and further to the individual nodes, the world map disappears and the datacenters/nodes are plotted in a circular layout. +- If you assign the latitude/longitude coordinates at the datacenter level, the Node Map shows the regions with single datacenters at the same location assigned to the datacenter, while regions with multiple datacenters are shown at the center of the datacenter coordinates in the region. When you drill down to the datacenter levels, the Node Map shows the datacenter at their assigned coordinates. Further drilling down to individual nodes shows the nodes in a circular layout. + +[Assign latitude/longitude coordinates](#step-3-set-the-latitudes-and-longitudes-for-the-localities) at the locality level that you want to view on the Node Map. + +## Known limitations + +### Unable to assign latitude/longitude coordinates to localities + +{% include {{ page.version.version }}/known-limitations/node-map.md %} + +### Displayed **Used Capacity** value is more than configured Capacity + +{% include {{ page.version.version }}/misc/available-capacity-metric.md %} + +## Location coordinates + +### AWS locations + +{% include {{ page.version.version }}/misc/aws-locations.md %} + +### Azure locations + +{% include {{ page.version.version }}/misc/azure-locations.md %} + +### Google Cloud locations + +{% include {{ page.version.version }}/misc/gce-locations.md %} diff --git a/v20.2/encryption.md b/v20.2/encryption.md new file mode 100644 index 00000000000..34c50ae076f --- /dev/null +++ b/v20.2/encryption.md @@ -0,0 +1,229 @@ +--- +title: Encryption +summary: Learn about the encryption features for secure CockroachDB clusters. +toc: true +--- + +Data encryption and decryption is the process of transforming plaintext data to cipher-text and vice versa using a key or password. + +## Encryption in flight + +CockroachDB uses TLS 1.2 for inter-node and client-node [authentication](authentication.html) as well as setting up a secure communication channel. Once the secure channel is set up, all inter-node and client-node network communication is encrypted using a [shared encryption key](https://en.wikipedia.org/wiki/Transport_Layer_Security) as per the TLS 1.2 protocol. This feature is enabled by default for all secure clusters and needs no additional configuration. + +## Encryption at Rest (Enterprise) + +Encryption at Rest provides transparent encryption of a node's data on the local disk. It allows encryption of all files on disk using [AES](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) in [counter mode](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)), with all key +sizes allowed. + +Encryption is performed in the [storage layer](architecture/storage-layer.html) and configured per store. +All files used by the store, regardless of contents, are encrypted with the desired algorithm. + +To allow arbitrary rotation schedules and ensure security of the keys, we use two layers of keys: + +- **Store keys** are provided by the user in a file. They are used to encrypt the list of data keys (see below). This is known as a **key encryption key**: its only purpose is to encrypt other keys. Store keys are never persisted by CockroachDB. Since very little data is encrypted using this key, it can have a very long lifetime without risk of reuse. + +- **Data keys** are automatically generated by CockroachDB. They are used to encrypt all files on disk. This is known as a **data encryption key**. Data keys are persisted in a key registry file, encrypted using the store key. The key has a short lifetime to avoid reuse. + +Store keys are specified at node startup by passing a path to a locally readable file. The file must contain 32 bytes (the key ID) followed by the key (16, 24, or 32 bytes). The size of the key dictates the version of AES to use (AES-128, AES-192, or AES-256). For an example showing how to create a store key, see [Generating key files](#generating-store-key-files) below. + +Also during node startup, CockroachDB uses a data key with the same length as the store key. If encryption has just been enabled, +the key size has changed, or the data key is too old (default lifetime is one week), CockroachDB generates a new data key. + +Any new file created by the store uses the currently-active data key. All data keys (both active and previous) are stored in a key registry file and encrypted with the active store key. + +After startup, if the active data key is too old, CockroachDB generates a new data key and marks it as active, using it for all further encryption. + +CockroachDB does not currently force re-encryption of older files but instead relies on normal RocksDB churn to slowly rewrite all data with the desired encryption. + +### Rotating keys + +Key rotation is necessary for Encryption at Rest for multiple reasons: + +- To prevent key reuse with the same encryption parameters (after encrypting many files). +- To reduce the risk of key exposure. + +Store keys are specified by the user and must be rotated by specifying different keys. +This is done by setting the `key` parameter of the `--enterprise-encryption` flag to the path to the new key, +and `old-key` to the previously-used key. + +Data keys will automatically be rotated at startup if any of the following conditions are met: + +- The active store key has changed. +- The encryption type has changed (different key size, or plaintext to/from encryption). +- The current data key is `rotation-period` old or more. + +Data keys will automatically be rotated at runtime if the current data key is `rotation-period` old or more. + +Once rotated, an old store key cannot be made the active key again. + +Upon store key rotation the data keys registry is decrypted using the old key and encrypted with the new +key. The newly-generated data key is used to encrypt all new data from this point on. + +### Changing encryption type + +The user can change the encryption type from plaintext to encryption, between different encryption algorithms (using various key sizes), or from encryption to plaintext. + +When changing the encryption type to plaintext, the data key registry is no longer encrypted and all previous data keys are readable by anyone. All data on the store is effectively readable. + +When changing from plaintext to encryption, it will take some time for all data to eventually be re-written and encrypted. + +### Recommendations + +There are a number of considerations to keep in mind when running with encryption. + +#### Deployment configuration + +To prevent key leakage, production deployments should: + +* Use encrypted swap, or disable swap entirely. +* Disable core files. + +CockroachDB attempts to disable core files at startup when encryption is requested, but it may fail. + +#### Key handling + +Key management is the most dangerous aspect of encryption. The following rules should be kept in mind: + +* Make sure that only the UNIX user running the `cockroach` process has access to the keys. +* Do not store the keys on the same partition/drive as the CockroachDB data. It is best to load keys at run time from a separate system (e.g., [Keywhiz](https://square.github.io/keywhiz/), [Vault](https://www.hashicorp.com/products/vault)). +* Rotate store keys frequently (every few weeks to months). +* Keep the data key rotation period low (default is one week). + +#### Other recommendations + +A few other recommendations apply for best security practices: + +* Do not switch from encrypted to plaintext, this leaks data keys. When plaintext is selected, all previously encrypted data must be considered reachable. +* Do not copy the encrypted files, as the data keys are not easily available. +* If encryption is desired, start a node with it enabled from the first run, without ever running in plaintext. + +{{site.data.alerts.callout_danger}} +Note that backups taken with the [`BACKUP`](backup.html) statement **are not encrypted** even if Encryption at Rest is enabled. Encryption at Rest only applies to the CockroachDB node's data on the local disk. If you want encrypted backups, you will need to encrypt your backup files using your preferred encryption method. +{{site.data.alerts.end}} + +### Examples + +#### Generating store key files + +Cockroach determines which encryption algorithm to use based on the size of the key file. +The key file must contain random data making up the key ID (32 bytes) and the actual key (16, 24, or 32 +bytes depending on the encryption algorithm). + +| Algorithm | Key size | Key file size | +|-|-|-| +| AES-128 | 128 bits (16 bytes) | 48 bytes | +| AES-192 | 192 bits (24 bytes) | 56 bytes | +| AES-256 | 256 bits (32 bytes) | 64 bytes | + +Generating a key file can be done using the `cockroach` CLI: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach gen encryption-key -s 128 /path/to/my/aes-128.key +~~~ + +Or the equivalent [openssl](https://www.openssl.org/docs/man1.0.2/apps/openssl.html) CLI command: + +{% include copy-clipboard.html %} +~~~ shell +$ openssl rand -out /path/to/my/aes-128.key 48 +~~~ + +#### Starting a node with encryption + +Encryption at Rest is configured at node start time using the `--enterprise-encryption` command line flag. +The flag specifies the encryption options for one of the stores on the node. If multiple stores exist, +the flag must be specified for each store. + +The flag takes the form: `--enterprise-encryption=path=,key=,old-key=,rotation-period=`. + +The allowed components in the flag are: + +| Component | Requirement | Description | +|-|-|-| +| `path` | Required | Path of the store to apply encryption to. | +| `key` | Required | Path to the key file to encrypt data with, or `plain` for plaintext. | +| `old-key` | Required | Path to the key file the data is encrypted with, or `plain` for plaintext. | +| `rotation-period` | Optional | How often data keys should be automatically rotated. Default: one week. | + +The `key` and `old-key` components must **always** be specified. They allow for transitions between +encryption algorithms, and between plaintext and encrypted. + +Starting a node for the first time using AES-128 encryption can be done using: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --store=cockroach-data --enterprise-encryption=path=cockroach-data,key=/path/to/my/aes-128.key,old-key=plain +~~~ + +{{site.data.alerts.callout_danger}} +Once specified for a given store, the `--enterprise-encryption` flag must always be present. +{{site.data.alerts.end}} + +#### Checking encryption status + +Encryption status can be seen on the node's stores report, reachable through: `http(s)://nodeaddress:8080/#/reports/stores/local` (or replace `local` with the node ID). For example, if you are running a [local cluster](secure-a-cluster.html), you can see the node's stores report at . + +The report shows encryption status for all stores on the selected node, including: + +* Encryption algorithm. +* Active store key information. +* Active data key information. +* The fraction of files/bytes encrypted using the active data key. + +CockroachDB relies on RocksDB compactions to write new files using the latest encryption key. It may take several days for all files to be replaced. Some files are only rewritten at startup, and some keep older copies around, requiring multiple restarts. You can force RocksDB compaction with the `cockroach debug compact` command (the node must first be [stopped](cockroach-quit.html)). + +Information about keys is written to [the logs](debug-and-error-logs.html), including: + +* Active/old key information at startup. +* New key information after data key rotation. + +Alternatively, you can use the [`cockroach debug encryption-active-key`](cockroach-debug-encryption-active-key.html) command to view information about a store's encryption algorithm and store key. + +#### Changing encryption algorithm or keys + +Encryption type and keys can be changed at any time by restarting the node. +To change keys or encryption type, the `key` component of the `--enterprise-encryption` flag is set to the new key, +while the key previously used must be specified in the `old-key` component. + +For example, we can switch from AES-128 to AES-256 using: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach start --store=cockroach-data --enterprise-encryption=path=cockroach-data,key=/path/to/my/aes-256.key,old-key=/path/to/my/aes-128.key +~~~ + +Upon starting, the node will read the existing data keys using the old encryption key (`aes-128.key`), then rewrite +the data keys using the new key (`aes-256.key`). A new data key will be generated to match the desired AES-256 algorithm. + +To check that the new key is active, use the stores report page in the Admin UI to [check the encryption status](#checking-encryption-status). + +To disable encryption, specify `key=plain`. The data keys will be stored in plaintext and new data will not be encrypted. + +To rotate keys, specify `key=/path/to/my/new-aes-128.key` and `old-key=/path/to/my/old-aes-128.key`. The data keys +will be decrypted using the old key and then encrypted using the new key. A new data key will also be generated. + +## Encrypted backups (Enterprise) + +{% include {{ page.version.version }}/backups/encrypted-backup-description.md %} + +## Encryption caveats + +### Higher CPU utilization + +Enabling Encryption at Rest might result in a higher CPU utilization. We estimate a 5-10% increase in CPU utilization. + +### Encryption for touchpoints with other services + +- S3 backup encryption +- Encrypted comms with Kafka + + +## See also + +- [Client Connection Parameters](connection-parameters.html) +- [Manual Deployment](manual-deployment.html) +- [Orchestrated Deployment](orchestration.html) +- [Local Deployment](secure-a-cluster.html) +- [Test Deployment](deploy-a-test-cluster.html) +- [Other Cockroach Commands](cockroach-commands.html) diff --git a/v20.2/enterprise-licensing.md b/v20.2/enterprise-licensing.md new file mode 100644 index 00000000000..f82861871d2 --- /dev/null +++ b/v20.2/enterprise-licensing.md @@ -0,0 +1,97 @@ +--- +title: Enterprise Features +summary: Request and set trial and enterprise license keys for CockroachDB +toc: true +--- + +CockroachDB distributes a single binary that contains both core and [enterprise features](https://www.cockroachlabs.com/pricing/). You can use core features without any license key. However, to use the enterprise features, you need either a trial or an enterprise license key. + +This page lists enterprise features, and shows you how to obtain and set trial and enterprise license keys for CockroachDB. + +## Enterprise features + +{% include {{ page.version.version }}/misc/enterprise-features.md %} + +## Types of licenses + +Type | Description +-------------|------------ +**Trial License** | A trial license enables you to try out CockroachDB enterprise features for 30 days for free. +**Enterprise License** | A paid enterprise license enables you to use CockroachDB enterprise features for longer periods (one year or more). + +{{site.data.alerts.callout_success}} +For quick local testing of Enterprise features, you can use the [`cockroach demo`](cockroach-demo.html) command, which starts a temporary, in-memory cluster with a SQL shell open and a trial license applied automatically. +{{site.data.alerts.end}} + +## Obtain a license + +To obtain a trial license, fill out [the registration form](https://www.cockroachlabs.com/get-cockroachdb/) and receive your trial license via email within a few minutes. + +To upgrade to an enterprise license, contact Sales. + +## Set a license + +As the CockroachDB `root` user, open the [built-in SQL shell](cockroach-sql.html) in insecure or secure mode, as per your CockroachDB setup. In the following example, we assume that CockroachDB is running in insecure mode. Then use the `SET CLUSTER SETTING` command to set the name of your organization and the license key: + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING cluster.organization = 'Acme Company'; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING enterprise.license = 'xxxxxxxxxxxx'; +~~~ + +## Verify a license + +To verify a license, open the [built-in SQL shell](cockroach-sql.html) and use the `SHOW CLUSTER SETTING` command to check the organization name and license key: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CLUSTER SETTING cluster.organization; +~~~ +~~~ + cluster.organization ++----------------------+ + Acme Company +(1 row) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CLUSTER SETTING enterprise.license; +~~~ +~~~ + enterprise.license ++-------------------------------------------+ + crl-0-ChB1x... +(1 row) +~~~ + +The license setting is also logged in the cockroach.log on the node where the command is run: + +{% include copy-clipboard.html %} +~~~ sql +$ cat cockroach.log | grep license +~~~ +~~~ +I171116 18:11:48.279604 1514 sql/event_log.go:102 [client=[::1]:56357,user=root,n1] Event: "set_cluster_setting", target: 0, info: {SettingName:enterprise.license Value:xxxxxxxxxxxx User:root} +~~~ + +## Renew an expired license + +After your license expires, the enterprise features stop working, but your production setup is unaffected. For example, the backup and restore features would not work until the license is renewed, but you would be able to continue using all other features of CockroachDB without interruption. + +To renew an expired license, contact Sales and then [set](enterprise-licensing.html#set-a-license) the new license. + +## See also + +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`SHOW CLUSTER SETTING`](show-cluster-setting.html) +- [Enterprise Trial –– Get Started](get-started-with-enterprise-trial.html) diff --git a/v20.2/error-handling-and-troubleshooting.md b/v20.2/error-handling-and-troubleshooting.md new file mode 100644 index 00000000000..d6e89397c1f --- /dev/null +++ b/v20.2/error-handling-and-troubleshooting.md @@ -0,0 +1,96 @@ +--- +title: Error Handling and Troubleshooting +summary: How to troubleshoot problems and handle transaction retry errors during application development +toc: true +--- + +This page has instructions for handling errors and troubleshooting problems that may arise during application development. + +## Troubleshooting query problems + +If you are not satisfied with your SQL query performance, follow the instructions in [Make Queries Fast][fast] to be sure you are avoiding common performance problems like full table scans, missing indexes, etc. + +If you have already optimized your SQL queries as described in [Make Queries Fast][fast] and are still having issues such as: + +- Hanging or "stuck" queries +- Queries that are slow some of the time (but not always) +- Low throughput of queries + +Take a look at [Troubleshoot SQL Behavior](query-behavior-troubleshooting.html). + +{{site.data.alerts.callout_info}} +If you aren't sure whether SQL query performance needs to be improved on your cluster, see [Identify slow queries](query-behavior-troubleshooting.html#identify-slow-queries). +{{site.data.alerts.end}} + +## Transaction retry errors + +Messages with the Postgres error code `40001` indicate that a transaction failed because it [conflicted with another concurrent or recent transaction accessing the same data](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention). The transaction needs to be retried by the client. + +If your language's client driver or ORM implements transaction retry logic internally (e.g., if you are using Python and [SQLAlchemy with the CockroachDB dialect](build-a-python-app-with-cockroachdb-sqlalchemy.html)), then you don't need to handle this logic from your application. + +If your driver or ORM does not implement this logic, then you will need to implement a retry loop in your application. + +{% include {{page.version.version}}/misc/client-side-intervention-example.md %} + +{{site.data.alerts.callout_info}} +If a consistently high percentage of your transactions are resulting in transaction retry errors, then you may need to evaluate your schema design and data access patterns to find and remove sources of contention. For more information, see [Understanding and Avoiding Transaction Contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention). +{{site.data.alerts.end}} + +For more information about transaction retry errors, see [Transaction retries](transactions.html#client-side-intervention). + +## Unsupported SQL features + +CockroachDB has support for [most SQL features](sql-feature-support.html). + +Additionally, CockroachDB supports [the PostgreSQL wire protocol and the majority of its syntax](postgresql-compatibility.html). This means that existing applications can often be migrated to CockroachDB without changing application code. + +However, you may encounter features of SQL or the Postgres dialect that are not supported by CockroachDB. For example, the following Postgres features are not supported: + +{% include {{page.version.version}}/sql/unsupported-postgres-features.md %} + +For more information about the differences between CockroachDB and Postgres feature support, see [PostgreSQL Compatibility](postgresql-compatibility.html). + +For more information about the SQL standard features supported by CockroachDB, see [SQL Feature Support](sql-feature-support.html) + +## Troubleshooting cluster problems + +As a developer, you will mostly be working with the CockroachDB [SQL API](sql-statements.html). + +However, you may need to access the underlying cluster to troubleshoot issues where the root cause is not your SQL, but something happening at the cluster level. Symptoms of cluster-level issues can include: + +- Cannot join a node to an existing cluster +- Networking, client connection, or authentication issues +- Clock sync, replication, or node liveness issues +- Capacity planning, storage, or memory issues +- Node decommissioning failures + +For more information about how to troubleshoot cluster-level issues, see [Troubleshoot Cluster Setup](cluster-setup-troubleshooting.html). + +## See also + +Reference information related to this page: + +- [Troubleshoot Query Behavior](query-behavior-troubleshooting.html) +- [Troubleshoot Cluster Setup](cluster-setup-troubleshooting.html) +- [Common errors](common-errors.html) +- [Transactions](transactions.html) +- [Transaction retries](transactions.html#client-side-intervention) +- [Understanding and Avoiding Transaction Contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) +- [SQL Layer][sql] + +Other common tasks: + +- [Connect to the Database](connect-to-the-database.html) +- [Insert Data](insert-data.html) +- [Query Data](query-data.html) +- [Update Data](update-data.html) +- [Delete Data](delete-data.html) +- [Run Multi-Statement Transactions](run-multi-statement-transactions.html) +- [Identify slow queries](query-behavior-troubleshooting.html#identify-slow-queries) +- [Make Queries Fast][fast] +- [Hello World Example apps](hello-world-example-apps.html) + + + +[sql]: architecture/sql-layer.html +[fast]: make-queries-fast.html diff --git a/v20.2/experimental-audit.md b/v20.2/experimental-audit.md new file mode 100644 index 00000000000..c7f3bab5a9c --- /dev/null +++ b/v20.2/experimental-audit.md @@ -0,0 +1,131 @@ +--- +title: EXPERIMENTAL_AUDIT +summary: Use the EXPERIMENTAL_AUDIT subcommand to turn SQL audit logging on or off for a table. +toc: true +--- + +`EXPERIMENTAL_AUDIT` is a subcommand of [`ALTER TABLE`](alter-table.html) that is used to turn [SQL audit logging](sql-audit-logging.html) on or off for a table. + +The audit logs contain detailed information about queries being executed against your system, including: + +- Full text of the query (which may include personally identifiable information (PII)) +- Date/Time +- Client address +- Application name + +For a detailed description of exactly what is logged, see the [Audit Log File Format](#audit-log-file-format) section below. + +{% include {{ page.version.version }}/misc/experimental-warning.md %} + +{% include {{ page.version.version }}/sql/combine-alter-table-commands.md %} + +## Synopsis + +
+{% include {{ page.version.version }}/sql/diagrams/experimental_audit.html %} +
+ +## Required privileges + +Only members of the `admin` role can enable audit logs on a table. By default, the `root` user belongs to the `admin` role. + +## Parameters + + Parameter | Description +--------------+---------------------------------------------------------- + `table_name` | The name of the table you want to create audit logs for. + `READ` | Log all table reads to the audit log file. + `WRITE` | Log all table writes to the audit log file. + `OFF` | Turn off audit logging. + +{{site.data.alerts.callout_info}} +As of version 2.0, this command logs all reads and writes, and both the READ and WRITE parameters are required (as shown in the examples below). In a future release, this should change to allow logging only reads, only writes, or both. +{{site.data.alerts.end}} + +## Audit log file format + +The audit log file format is as shown below. The numbers above each column are not part of the format; they correspond to the descriptions that follow. + +~~~ +[1] [2] [3] [4] [5] [6] [7a] [7b] [7c] [7d] [7e] [7f] [7g] [7h] [7i] +I180211 07:30:48.832004 317 sql/exec_log.go:90 [client=127.0.0.1:62503, user=root, n1] 13 exec "cockroach" {"ab"[53]:READ} "SELECT nonexistent FROM ab" {} 0.123 12 ERROR 0 +~~~ + +1. Date +2. Time (in UTC) +3. Goroutine ID. This column is used for troubleshooting CockroachDB and may change its meaning at any time. +4. Where the log line was generated +5. Logging tags (the contents and order may vary) + - Client address + - Username + - Node ID +6. Log entry counter +7. Log message: + - a. Label indicating where the data was generated (useful for troubleshooting) + - b. Current value of the [`application_name`](set-vars.html) session setting + - c. Logging trigger: + - The list of triggering tables and access modes for audit logs, since only certain (read/write) activities are added to the audit log + - d. Full text of the query (Note: May contain PII) + - e. Placeholder values, if any + - f. Query execution time (in milliseconds) + - g. Number of rows produced (e.g., for `SELECT`) or processed (e.g., for `INSERT` or `UPDATE`). + - h. Status of the query + - `OK` for success + - `ERROR` otherwise + - i. Number of times the statement was [retried automatically](transactions.html#automatic-retries) by the server so far. + +## Audit log file storage location + +By default, audit logs are stored in the [same directory](debug-and-error-logs.html#write-to-file) as the other logs generated by CockroachDB. + +To store the audit log files in a specific directory, pass the `--sql-audit-dir` flag to [`cockroach start`](cockroach-start.html). + +{{site.data.alerts.callout_success}} +If your deployment requires particular lifecycle and access policies for audit log files, point `--sql-audit-dir` at a directory that has permissions set so that only CockroachDB can create/delete files. +{{site.data.alerts.end}} + +## Viewing schema changes + +{% include {{ page.version.version }}/misc/schema-change-view-job.md %} + +## Examples + +### Turn on audit logging + +Let's say you have a `customers` table that contains personally identifiable information (PII). To turn on audit logs for that table, run the following command: + +{% include copy-clipboard.html %} +~~~ sql +ALTER TABLE customers EXPERIMENTAL_AUDIT SET READ WRITE; +~~~ + +Now, every access of customer data is added to the audit log with a line that looks like the following: + +~~~ +I180211 07:30:48.832004 317 sql/exec_log.go:90 [client=127.0.0.1:62503,user=root,n1] 13 exec "cockroach" {"customers"[53]:READ} "SELECT * FROM customers" {} 123.45 12 OK +I180211 07:30:48.832004 317 sql/exec_log.go:90 [client=127.0.0.1:62503,user=root,n1] 13 exec "cockroach" {"customers"[53]:READ} "SELECT nonexistent FROM customers" {} 0.123 12 ERROR +~~~ + +To turn on auditing for more than one table, issue a separate `ALTER` statement for each table. + +For a description of the log file format, see the [Audit Log File Format](#audit-log-file-format) section. + +{{site.data.alerts.callout_success}} +For a more detailed example, see [SQL Audit Logging](sql-audit-logging.html). +{{site.data.alerts.end}} + +### Turn off audit logging + +To turn off logging, issue the following command: + +{% include copy-clipboard.html %} +~~~ sql +ALTER TABLE customers EXPERIMENTAL_AUDIT SET OFF; +~~~ + +## See also + +- [SQL Audit Logging](sql-audit-logging.html) +- [`ALTER TABLE`](alter-table.html) +- [`cockroach start` logging flags](cockroach-start.html) +- [`SHOW JOBS`](show-jobs.html) diff --git a/v20.2/experimental-features.md b/v20.2/experimental-features.md new file mode 100644 index 00000000000..8493b4b6593 --- /dev/null +++ b/v20.2/experimental-features.md @@ -0,0 +1,153 @@ +--- +title: Experimental Features +summary: Learn about the experimental features available in CockroachDB +toc: true +--- + +This page lists the experimental features that are available in CockroachDB {{ page.version.version }}. + +{{site.data.alerts.callout_danger}} +**This page describes experimental features.** Their interfaces and outputs are subject to change, and there may be bugs. +
+
+If you encounter a bug, please [file an issue](file-an-issue.html). +{{site.data.alerts.end}} + +## Session variables + +The table below lists the experimental session settings that are available. For a complete list of session variables, see [`SHOW` (session settings)](show-vars.html). + +| Variable | Default Value | Description | +|-------------------------------------+---------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `experimental_enable_hash_sharded_indexes` | `'off'` | If set to `'on'`, enables [hash-sharded indexes](#hash-sharded-indexes) with `USING HASH`. | +| `experimental_enable_temp_tables` | `'off'` | If set to `'on'`, enables [temporary objects](#temporary-objects), including [temporary tables](temporary-tables.html), [temporary views](views.html#temporary-views), and [temporary sequences](create-sequence.html#temporary-sequences). | +| `experimental_serial_normalization` | `'rowid'` | If set to `'virtual_sequence'`, make the [`SERIAL`](serial.html) pseudo-type optionally auto-create a sequence for [better compatibility with Hibernate sequences](https://forum.cockroachlabs.com/t/hibernate-sequence-generator-returns-negative-number-and-ignore-unique-rowid/). | + +## SQL statements + +### Keep SQL audit logs + +Log queries against a table to a file. For more information, see [`ALTER TABLE ... EXPERIMENTAL_AUDIT`](experimental-audit.html). + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t EXPERIMENTAL_AUDIT SET READ WRITE; +~~~ + +### Relocate leases and replicas + +You have the following options for controlling lease and replica location: + +1. Relocate leases and replicas using `EXPERIMENTAL_RELOCATE` +2. Relocate just leases using `EXPERIMENTAL_RELOCATE LEASE` + +For example, to distribute leases and ranges for N primary keys across N stores in the cluster, run a statement with the following structure: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t EXPERIMENTAL_RELOCATE SELECT ARRAY[, , ..., ], , , ..., ; +~~~ + +To relocate just the lease without moving the replicas, run a statement like the one shown below, which moves the lease for the range containing primary key 'foo' to store 1. + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE t EXPERIMENTAL_RELOCATE LEASE SELECT 1, 'foo'; +~~~ + +### Show table fingerprints + +Table fingerprints are used to compute an identification string of an entire table, for the purpose of gauging whether two tables have the same data. This is useful, for example, when restoring a table from backup. + +Example: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE t; +~~~ + +~~~ + index_name | fingerprint +------------+--------------------- + primary | 1999042440040364641 +(1 row) +~~~ + +### Turn on KV event tracing + +Use session tracing (via [`SHOW TRACE FOR SESSION`](show-trace.html)) to report the replicas of all KV events that occur during its execution. + +Example: + +{% include copy-clipboard.html %} +~~~ sql +> SET tracing = on; +> SELECT * from t; +> SET tracing = off; +> SHOW EXPERIMENTAL_REPLICA TRACE FOR SESSION; +~~~ + +~~~ + timestamp | node_id | store_id | replica_id +----------------------------------+---------+----------+------------ + 2018-10-18 15:50:13.345879+00:00 | 3 | 3 | 7 + 2018-10-18 15:50:20.628383+00:00 | 2 | 2 | 26 +~~~ + +### Check for constraint violations with `SCRUB` + +Checks the consistency of [`UNIQUE`](unique.html) indexes, [`CHECK`](check.html) constraints, and more. Partially implemented; see [cockroachdb/cockroach#10425](https://github.com/cockroachdb/cockroach/issues/10425) for details. + +{{site.data.alerts.callout_info}} +This example uses the `users` table from our open-source, fictional peer-to-peer vehicle-sharing application, [MovR](movr.html). +{{site.data.alerts.end}} + +{% include copy-clipboard.html %} +~~~ sql +> EXPERIMENTAL SCRUB table movr.users; +~~~ + +~~~ + job_uuid | error_type | database | table | primary_key | timestamp | repaired | details +----------+--------------------------+----------+-------+----------------------------------------------------------+---------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + | index_key_decoding_error | movr | users | ('boston','0009eeb5-d779-4bf8-b1bd-8566533b105c') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'06484 Christine Villages\\nGrantport, TN 01572'", "city": "'boston'", "credit_card": "'4634253150884'", "id": "'0009eeb5-d779-4bf8-b1bd-8566533b105c'", "name": "'Jessica Webb'"}} + | index_key_decoding_error | movr | users | ('los angeles','0001252c-fc16-4006-b6dc-c6b1a0fd1f5b') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'91309 Warner Springs\\nLake Danielmouth, PR 33400'", "city": "'los angeles'", "credit_card": "'3584736360686445'", "id": "'0001252c-fc16-4006-b6dc-c6b1a0fd1f5b'", "name": "'Rebecca Gibson'"}} + | index_key_decoding_error | movr | users | ('new york','000169a5-e337-4441-b664-dae63e682980') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'0787 Christopher Highway Apt. 363\\nHamptonmouth, TX 91864-2620'", "city": "'new york'", "credit_card": "'4578562547256688'", "id": "'000169a5-e337-4441-b664-dae63e682980'", "name": "'Christopher Johnson'"}} + | index_key_decoding_error | movr | users | ('paris','00089fc4-e5b1-48f6-9f0b-409905f228c4') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'46735 Martin Summit\\nMichaelview, OH 10906-5889'", "city": "'paris'", "credit_card": "'5102207609888778'", "id": "'00089fc4-e5b1-48f6-9f0b-409905f228c4'", "name": "'Nicole Fuller'"}} + | index_key_decoding_error | movr | users | ('rome','000209fc-69a1-4dd5-8053-3b5e5769876d') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'473 Barrera Vista Apt. 890\\nYeseniaburgh, CO 78087'", "city": "'rome'", "credit_card": "'3534605564661093'", "id": "'000209fc-69a1-4dd5-8053-3b5e5769876d'", "name": "'Sheryl Shea'"}} + | index_key_decoding_error | movr | users | ('san francisco','00058767-1e83-4e18-999f-13b5a74d7225') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'5664 Acevedo Drive Suite 829\\nHernandezview, MI 13516'", "city": "'san francisco'", "credit_card": "'376185496850202'", "id": "'00058767-1e83-4e18-999f-13b5a74d7225'", "name": "'Kevin Turner'"}} + | index_key_decoding_error | movr | users | ('seattle','0002e904-1256-4528-8b5f-abad16e695ff') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'81499 Samuel Crescent Suite 631\\nLake Christopherborough, PR 50401'", "city": "'seattle'", "credit_card": "'38743493725890'", "id": "'0002e904-1256-4528-8b5f-abad16e695ff'", "name": "'Mark Williams'"}} + | index_key_decoding_error | movr | users | ('washington dc','00007caf-2014-4696-85b0-840e7d8b6db9') | 2018-10-18 16:00:38.65916 | f | {"error_message": "key ordering did not match datum ordering. IndexDescriptor=ASC", "index_name": "primary", "row_data": {"address": "e'4578 Holder Trafficway\\nReynoldsside, IL 23520-7418'", "city": "'washington dc'", "credit_card": "'30454993082943'", "id": "'00007caf-2014-4696-85b0-840e7d8b6db9'", "name": "'Marie Miller'"}} +(8 rows) +~~~ + +### Show range information for a specific row + +The [`SHOW RANGE ... FOR ROW`](show-range-for-row.html) statement shows information about a [range](architecture/overview.html#glossary) for a particular row of data. This information is useful for verifying how SQL data maps to underlying ranges, and where the replicas for a range are located. + +## Functions and Operators + +The table below lists the experimental SQL functions and operators available in CockroachDB. For more information, see each function's documentation at [Functions and Operators](functions-and-operators.html). + +| Function | Description | +|----------------------------------------------------------------------------------+-------------------------------------------------| +| [`experimental_strftime`](functions-and-operators.html#date-and-time-functions) | Format time using standard `strftime` notation. | +| [`experimental_strptime`](functions-and-operators.html#date-and-time-functions) | Format time using standard `strptime` notation. | +| [`experimental_uuid_v4()`](functions-and-operators.html#id-generation-functions) | Return a UUID. | + + +## Temporary objects + + Support for [temporary tables](temporary-tables.html), [temporary views](views.html#temporary-views), and [temporary sequences](create-sequence.html#temporary-sequences) is currently experimental in CockroachDB. If you create too many temporary objects in a session, the performance of DDL operations will degrade. Performance limitations could persist long after creating the temporary objects. For more details, see [cockroachdb/cockroach#46260](https://github.com/cockroachdb/cockroach/issues/46260). + +## Hash-sharded indexes + + CockroachDB supports hash-sharded indexes with the [`USING HASH`](create-index.html#parameters) keywords. Hash-sharded indexes distribute sequential traffic uniformly across ranges, eliminating single-range hotspots and improving write performance on sequentially-keyed indexes at a small cost to read performance. For more information, see [Hash-sharded indexes](indexes.html#hash-sharded-indexes). + +## See Also + +- [`SHOW` (session)](show-vars.html) +- [Functions and Operators](functions-and-operators.html) +- [`ALTER TABLE ... EXPERIMENTAL_AUDIT`](experimental-audit.html) +- [`SHOW TRACE FOR SESSION`](show-trace.html) +- [`SHOW RANGE ... FOR ROW`](show-range-for-row.html) diff --git a/v20.2/explain-analyze.md b/v20.2/explain-analyze.md new file mode 100644 index 00000000000..fec630025ec --- /dev/null +++ b/v20.2/explain-analyze.md @@ -0,0 +1,164 @@ +--- +title: EXPLAIN ANALYZE +summary: The EXPLAIN ANALYZE statement executes a query and generates a physical query plan with execution statistics. +toc: true +--- + +The `EXPLAIN ANALYZE` [statement](sql-statements.html) **executes a SQL query** and generates a URL for a physical query plan with execution statistics, or a URL to download a bundle with more details about the query plan. Query plans provide information around SQL execution, which can be used to troubleshoot slow queries by figuring out where time is being spent, how long a processor (i.e., a component that takes streams of input rows and processes them according to a specification) is not doing work, etc. For more information about distributed SQL queries, see the [DistSQL section of our SQL Layer Architecture docs](architecture/sql-layer.html#distsql). + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/sql/physical-plan-url.md %} +{{site.data.alerts.end}} + +## Aliases + +In CockroachDB, the following are aliases for `EXPLAIN ANALYZE`: + +- `EXPLAIN ANALYSE` + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/explain_analyze.html %}
+ +## Parameters + +Parameter | Description +-------------------|----------- +`DISTSQL` | _(Default)_ Generate a link to a distributed SQL physical query plan tree. For more information, see [Default option](#default-option). +`DEBUG` | Generate a ZIP file containing files with detailed information about the query and the database objects referenced in the query. For more information, see [`DEBUG` option](#debug-option). +`preparable_stmt` | The [statement](sql-grammar.html#preparable_stmt) you want to execute and analyze. All preparable statements are explainable. + +## Required privileges + +The user requires the appropriate [privileges](authorization.html#assign-privileges) for the statement being explained. + +## Success responses + +Successful `EXPLAIN ANALYZE` (and `EXPLAIN ANALYZE (DISTSQL)`) statements return a table with the following columns: + + Column | Description +--------|------------ +**automatic** | If `true`, the query is distributed. For more information about distributed SQL queries, see the [DistSQL section of our SQL Layer Architecture docs](architecture/sql-layer.html#distsql). +**url** | The URL generated for a physical query plan that provides high level information about how a query will be executed. For details about reading the physical query plan, see [DistSQL Plan Viewer](#distsql-plan-viewer).

{% include {{ page.version.version }}/sql/physical-plan-url.md %} + +If you use the [`DEBUG` option](#debug-option), the statement will return a single `text` column with a URL and instructions to download the `DEBUG` bundle, which includes the physical query plan. + +## Default option + +By default, `EXPLAIN ANALYZE` uses the `DISTQL` option, which generates a physical query plan diagram in the [DistSQL Plan Viewer](#distsql-plan-viewer). `EXPLAIN ANALYZE` and `EXPLAIN ANALYZE (DISTSQL)` produce the same output. + +### DistSQL Plan Viewer + +The DistSQL Plan Viewer displays the physical query plan, as well as execution statistics: + +Field | Description +------+------------ +<ProcessorName>/<n> | The processor and processor ID used to read data into the SQL execution engine.

A processor is a component that takes streams of input rows, processes them according to a specification, and outputs one stream of rows. For example, an "aggregator" aggregates input rows. +<index>@<table> | The index used. +Out | The output columns. +@<n> | The index of the column relative to the input. +Render | The stage that renders the output. +unordered / ordered | _(Blue box)_ A synchronizer that takes one or more output streams and merges them to be consumable by a processor. An ordered synchronizer is used to merge ordered streams and keeps the rows in sorted order. +<data type> | If [`EXPLAIN(DISTSQL, TYPES)`](explain.html#distsql-option) is specified, lists the data types of the input columns. +left(@<n>)=right(@<n>) | The equality columns used in the join. +rows read | The number of rows read by the processor. +stall time | How long the processor spent not doing work. This is aggregated into the stall time numbers as the query progresses down the tree (i.e., stall time is added up and overlaps with previous time). +stored side | The smaller table that was stored as an in-memory hash table. +max memory used | How much memory (if any) is used to buffer rows. +by hash | _(Orange box)_ The router, which is a component that takes one stream of input rows and sends them to a node according to a routing algorithm.

For example, a hash router hashes columns of a row and sends the results to the node that is aggregating the result rows. +max disk used | How much disk (if any) is used to buffer rows. Routers and processors will spill to disk buffering if there is not enough memory to buffer the rows. +rows routed | How many rows were sent by routers, which can be used to understand network usage. +bytes sent | The number of actual bytes sent (i.e., encoding of the rows). This is only relevant when doing network communication. +Response | The response back to the client. + +{{site.data.alerts.callout_info}} +Any or all of the above fields may display for a given query plan. +{{site.data.alerts.end}} + +## `DEBUG` option + + `EXPLAIN ANALYZE (DEBUG)` executes a query and generates a link to a ZIP file that contains the [physical query plan](#distsql-plan-viewer), execution statistics, statement tracing, and other information about the query. + + File | Description +--------------------+------------------- +`stats-
.sql` | Contains [statistics](create-statistics.html) for a table in the query. +`schema.sql` | Contains [`CREATE`](create-table.html) statements for objects in the query. +`env.sql` | Contains information about the CockroachDB environment. +`trace.json` | Contains [statement traces](show-trace.html). +`distsql.html` | The query's [physical query plan](#distsql-plan-viewer). This diagram is identical to the one generated by [`EXPLAIN(DISTSQL)`](explain.html#distsql-option) +`plan.txt` | The query execution plan. This is identical to the output of [`EXPLAIN (VERBOSE)`](explain.html#verbose-option). +`opt-vv.txt` | The query plan tree generated by the [cost-based optimizer](cost-based-optimizer.html), with cost details and input column data types. This is identical to the output of [`EXPLAIN (OPT, TYPES)`](explain.html#opt-option). +`opt-v.txt` | The query plan tree generated by the [cost-based optimizer](cost-based-optimizer.html), with cost details. This is identical to the output of [`EXPLAIN (OPT, VERBOSE)`](explain.html#opt-option). +`opt.txt` | The query plan tree generated by the [cost-based optimizer](cost-based-optimizer.html). This is identical to the output of [`EXPLAIN (OPT)`](explain.html#opt-option). +`statement.txt` | The SQL statement for the query. + +You can obtain this ZIP file by following the link provided in the `EXPLAIN ANALYZE (DEBUG)` output, or by navigating to **Advanced Debug** -> **Statement Diagnostics History** in the [Admin UI](admin-ui-overview.html). + +## Examples + +### `EXPLAIN ANALYZE` + +Use `EXPLAIN ANALYZE` without an option, or equivalently with the `DISTSQL` option, to execute a query and generate a link to a physical query plan with execution statistics. + +For example, the following `EXPLAIN ANALYZE` statement executes a simple query against the [TPC-H database](http://www.tpc.org/tpch/) loaded to a 3-node CockroachDB cluster, and then generates a link to a physical query plan with execution statistics: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN ANALYZE SELECT l_shipmode, AVG(l_extendedprice) FROM lineitem GROUP BY l_shipmode; +~~~ + +~~~ + automatic | url +------------+----------------------------------------------- + true | https://cockroachdb.github.io/distsqlplan... +~~~ + +To view the [DistSQL Plan Viewer](#distsql-plan-viewer), point your browser to the URL provided: + +EXPLAIN ANALYZE (DISTSQL) + + +### `EXPLAIN ANALYZE (DEBUG)` + +Use the [`DEBUG`](#debug-option) option to generate a ZIP file containing files with information about the query and the database objects referenced in the query. For example: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN ANALYZE (DEBUG) SELECT l_shipmode, AVG(l_extendedprice) FROM lineitem GROUP BY l_shipmode; +~~~ + +~~~ + text +-------------------------------------------------------------------------------- + Statement diagnostics bundle generated. Download from the Admin UI (Advanced + Debug -> Statement Diagnostics History) or use the direct link below. + Admin UI: http://127.0.0.1:12345 + Direct link: http://127.0.0.1:12345/_admin/v1/stmtbundle/... +~~~ + +Navigating to the URL will automatically download the ZIP file. As the output suggests, you can also obtain the bundle by navigating to the **Statement Diagnostics History** page in the [Admin UI](admin-ui-overview.html). + +## See also + +- [`ALTER TABLE`](alter-table.html) +- [`ALTER SEQUENCE`](alter-sequence.html) +- [`BACKUP`](backup.html) +- [`CANCEL JOB`](cancel-job.html) +- [`CREATE DATABASE`](create-database.html) +- [`DROP DATABASE`](drop-database.html) +- [`EXPLAIN`](explain.html) +- [`EXECUTE`](sql-grammar.html#execute_stmt) +- [`IMPORT`](import.html) +- [Indexes](indexes.html) +- [`INSERT`](insert.html) +- [`PAUSE JOB`](pause-job.html) +- [`RESET`](reset-vars.html) +- [`RESTORE`](restore.html) +- [`RESUME JOB`](resume-job.html) +- [`SELECT`](select-clause.html) +- [Selection Queries](selection-queries.html) +- [`SET`](set-vars.html) +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`SHOW COLUMNS`](show-columns.html) +- [`UPDATE`](update.html) +- [`UPSERT`](upsert.html) diff --git a/v20.2/explain.md b/v20.2/explain.md new file mode 100644 index 00000000000..226977fb5b7 --- /dev/null +++ b/v20.2/explain.md @@ -0,0 +1,622 @@ +--- +title: EXPLAIN +summary: The EXPLAIN statement provides information you can use to optimize SQL queries. +toc: true +--- + +The `EXPLAIN` [statement](sql-statements.html) returns CockroachDB's query plan for an [explainable statement](sql-grammar.html#preparable_stmt). You can then use this information to optimize the query. + +{{site.data.alerts.callout_success}} +To actually execute a statement and return a physical query plan with execution statistics, use [`EXPLAIN ANALYZE`](explain-analyze.html). +{{site.data.alerts.end}} + +## Query optimization + +Using `EXPLAIN`'s output, you can optimize your queries by taking the following points into consideration: + +- Queries with fewer levels execute more quickly. Restructuring queries to require fewer levels of processing will generally improve performance. + +- Avoid scanning an entire table, which is the slowest way to access data. You can avoid this by [creating indexes](indexes.html) that contain at least one of the columns that the query is filtering in its `WHERE` clause. + +- By default, the [vectorized execution](vectorized-execution.html) engine is enabled for all [supported operations](vectorized-execution.html#disk-spilling-operations) and [data types](vectorized-execution.html#supported-data-types). If you are querying a table with a small number of rows, it might be more efficient to use row-oriented execution. The `vectorize_row_count_threshold` [cluster setting](cluster-settings.html) specifies the minimum number of rows required to use the vectorized engine to execute a query plan. + +You can find out if your queries are performing entire table scans by using `EXPLAIN` to see which: + +- Indexes the query uses; shown as the **Description** value of rows with the **Field** value of `table` + +- Key values in the index are being scanned; shown as the **Description** value of rows with the **Field** value of `spans` + +For more information, see [Find the Indexes and Key Ranges a Query Uses](#find-the-indexes-and-key-ranges-a-query-uses). + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/explain.html %}
+ +## Required privileges + +The user requires the appropriate [privileges](authorization.html#assign-privileges) for the statement being explained. + +## Parameters + + Parameter | Description +--------------------+------------ + `VERBOSE` | Show as much information as possible about the query plan. + `TYPES` | Include the intermediate [data types](data-types.html) CockroachDB chooses to evaluate intermediate SQL expressions. + `OPT` | Display the query plan tree generated by the [cost-based optimizer](cost-based-optimizer.html).

To include cost details used by the optimizer in planning the query, use `OPT, VERBOSE`. To include cost and type details, use `OPT, TYPES`. To include all details used by the optimizer, including statistics, use `OPT, ENV`. + `VEC` | Show detailed information about the [vectorized execution](vectorized-execution.html) plan for a query. If the table queried includes [unsupported data types](vectorized-execution.html#supported-data-types), an unhandled data type error is returned. + `preparable_stmt` | The [statement](sql-grammar.html#preparable_stmt) you want details about. All preparable statements are explainable. + `DISTSQL` | Generate a URL to a [distributed SQL physical query plan tree](explain-analyze.html#distsql-plan-viewer).

{% include {{ page.version.version }}/sql/physical-plan-url.md %} + +{{site.data.alerts.callout_danger}} +`EXPLAIN` also includes other modes besides query plans that are useful only to CockroachDB developers, which are not documented here. +{{site.data.alerts.end}} + +## Success responses + +Successful `EXPLAIN` statements return tables with the following columns: + + Column | Description +-----------|------------- +**Tree** | A tree representation of the hierarchy of the query plan. +**Field** | The name of a property for the query plan.

The `distributed` and `vectorized` properties apply to the entire query plan. All other properties apply to the query plan node in the **Tree** column. +**Description** | Additional information about the parameter in **Field**. +**Columns** | The columns provided to the processes at lower levels of the hierarchy. Included in `TYPES` and `VERBOSE` output. +**Ordering** | The order in which results are presented to the processes at each level of the hierarchy, as well as other properties of the result set at each level. Included in `TYPES` and `VERBOSE` output. + +## Examples + +The following examples use the [`startrek` example dataset](cockroach-demo.html#datasets). To follow along, you can use `cockroach demo startrek` to start a temporary, in-memory cluster with the `startrek` dataset preloaded. + +### Default query plans + +By default, `EXPLAIN` includes the least detail about the query plan but can be useful to find out which indexes and index key ranges are used by a query. For example: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + tree | field | description +------------+-------------+------------------- + | distributed | true + | vectorized | false + sort | | + │ | order | +season + └── scan | | + | table | episodes@primary + | spans | FULL SCAN + | filter | season > 3 +(8 rows) +~~~ + +The `tree` column of the output shows the tree structure of the query plan, in this case a `sort` and then a `scan`. + +The `field` and `description` columns describe a set of properties, some global to the query, and some specific to an operation listed in the `tree` column (in this case, `sort` or `scan`): + +- `distributed`:`true` +
The query plan will be distributed to multiple nodes on the cluster. +- `vectorized`:`false` +
The plan will be executed with the row-oriented execution engine, and not the [vectorized engine](vectorized-execution.html). +- `order`:`+season` +
The sort will be ordered ascending on the `season` column. +- `table`:`episodes@primary` +
The table is scanned on the `primary` index. +- `spans`:`FULL SCAN` +
The table is scanned on all key ranges of the `primary` index (i.e., a full table scan). For more information on indexes and key ranges, see the [example](#find-the-indexes-and-key-ranges-a-query-uses) below. +- `filter`: `season > 3` +
The scan filters on the `season` column. + +If you run `EXPLAIN` on a [join](joins.html) query, the output will display which type of join will be executed. For example, the following `EXPLAIN` output shows that the query will perform a [hash join](joins.html#hash-joins): + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM quotes AS q +JOIN episodes AS e ON q.episode = e.id; +~~~ + +~~~ + tree | field | description +------------+--------------------+------------------- + | distributed | true + | vectorized | false + hash-join | | + │ | type | inner + │ | equality | (episode) = (id) + │ | right cols are key | + ├── scan | | + │ | table | quotes@primary + │ | spans | FULL SCAN + └── scan | | + | table | episodes@primary + | spans | FULL SCAN +(12 rows) +~~~ + +And the following output shows that the query will perform a simple cross join: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM quotes AS q +JOIN episodes AS e ON q.episode = '2'; +~~~ + +~~~ + tree | field | description +-----------------------+-------------+---------------------------- + | distributed | true + | vectorized | false + render | | + └── cross-join | | + │ | type | cross + ├── scan | | + │ | table | episodes@primary + │ | spans | FULL SCAN + └── index-join | | + │ | table | quotes@primary + │ | key columns | rowid + └── scan | | + | table | quotes@quotes_episode_idx + | spans | /2-/3 +(14 rows) +~~~ + +### `VERBOSE` option + +The `VERBOSE` option: + +- Includes SQL expressions that are involved in each processing stage, providing more granular detail about which portion of your query is represented at each level. +- Includes detail about which columns are being used by each level, as well as properties of the result set on that level. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (VERBOSE) SELECT * FROM quotes AS q +JOIN episodes AS e ON q.episode = e.id +WHERE e.season = '1' +ORDER BY e.stardate ASC; +~~~ + +The output of [`EXPLAIN`](explain.html#verbose-option) also shows whether `equality cols are key` for [lookup joins](joins.html#lookup-joins), which means that the lookup columns form a key in the target table such that each lookup has at most one result. + +~~~ + tree | field | description | columns | ordering +-----------------+--------------------+------------------+--------------------------------------------------------------------------+------------ + | distributed | true | | + | vectorized | false | | + sort | | | (quote, characters, stardate, episode, id, season, num, title, stardate) | +stardate + │ | order | +stardate | | + └── hash-join | | | (quote, characters, stardate, episode, id, season, num, title, stardate) | + │ | type | inner | | + │ | equality | (episode) = (id) | | + │ | right cols are key | | | + ├── scan | | | (quote, characters, stardate, episode) | + │ | table | quotes@primary | | + │ | spans | FULL SCAN | | + └── scan | | | (id, season, num, title, stardate) | + | table | episodes@primary | | + | spans | FULL SCAN | | + | filter | season = 1 | | +(15 rows) +~~~ + +### `TYPES` option + +The `TYPES` mode includes the types of the values used in the query plan. It also includes the SQL expressions that were involved in each processing stage, and includes the columns used by each level. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (TYPES) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + tree | field | description | columns | ordering +------------+-------------+----------------------------------+---------------------------------------------------------------+----------- + | distributed | true | | + | vectorized | false | | + sort | | | (id int, season int, num int, title string, stardate decimal) | +season + │ | order | +season | | + └── scan | | | (id int, season int, num int, title string, stardate decimal) | + | table | episodes@primary | | + | spans | FULL SCAN | | + | filter | ((season)[int] > (3)[int])[bool] | | +(8 rows) +~~~ + +### `OPT` option + +The `OPT` option displays the query plan tree generated by the [cost-based optimizer](cost-based-optimizer.html). For example: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + text +----------------------------- + sort + └── select + ├── scan episodes + └── filters + └── season > 3 +(5 rows) +~~~ + + + +To include cost details used by the optimizer in planning the query, use `OPT, VERBOSE`: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT, VERBOSE) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + text +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + sort + ├── columns: id:1 season:2 num:3 title:4 stardate:5 + ├── stats: [rows=26.3333333, distinct(2)=1, null(2)=0] + ├── cost: 98.6419109 + ├── key: (1) + ├── fd: (1)-->(2-5) + ├── ordering: +2 + ├── prune: (1,3-5) + ├── interesting orderings: (+1) + └── select + ├── columns: id:1 season:2 num:3 title:4 stardate:5 + ├── stats: [rows=26.3333333, distinct(2)=1, null(2)=0] + ├── cost: 95.62 + ├── key: (1) + ├── fd: (1)-->(2-5) + ├── prune: (1,3-5) + ├── interesting orderings: (+1) + ├── scan episodes + │ ├── columns: id:1 season:2 num:3 title:4 stardate:5 + │ ├── stats: [rows=79, distinct(1)=79, null(1)=0, distinct(2)=3, null(2)=0] + │ │ histogram(1)= 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 + │ │ <--- 1 --- 2 --- 3 --- 4 --- 5 --- 6 --- 7 --- 8 --- 9 --- 10 --- 11 --- 12 --- 13 --- 14 --- 15 --- 16 --- 17 --- 18 --- 19 --- 20 --- 21 --- 22 --- 23 --- 24 --- 25 --- 26 --- 27 --- 28 --- 29 --- 30 --- 31 --- 32 --- 33 --- 34 --- 35 --- 36 --- 37 --- 38 --- 39 --- 40 --- 41 --- 42 --- 43 --- 44 --- 45 --- 46 --- 47 --- 48 --- 49 --- 50 --- 51 --- 52 --- 53 --- 54 --- 55 --- 56 --- 57 --- 58 --- 59 --- 60 --- 61 --- 62 --- 63 --- 64 --- 65 --- 66 --- 67 --- 68 --- 69 --- 70 --- 71 --- 72 --- 73 --- 74 --- 75 --- 76 --- 77 --- 78 --- 79 + │ ├── cost: 94.82 + │ ├── key: (1) + │ ├── fd: (1)-->(2-5) + │ ├── prune: (1-5) + │ └── interesting orderings: (+1) + └── filters + └── season:2 > 3 [outer=(2), constraints=(/2: [/4 - ]; tight)] +(29 rows) +~~~ + + + +To include cost and type details, use `OPT, TYPES`: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT, TYPES) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + text +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + sort + ├── columns: id:1(int!null) season:2(int!null) num:3(int) title:4(string) stardate:5(decimal) + ├── stats: [rows=26.3333333, distinct(2)=1, null(2)=0] + ├── cost: 98.6419109 + ├── key: (1) + ├── fd: (1)-->(2-5) + ├── ordering: +2 + ├── prune: (1,3-5) + ├── interesting orderings: (+1) + └── select + ├── columns: id:1(int!null) season:2(int!null) num:3(int) title:4(string) stardate:5(decimal) + ├── stats: [rows=26.3333333, distinct(2)=1, null(2)=0] + ├── cost: 95.62 + ├── key: (1) + ├── fd: (1)-->(2-5) + ├── prune: (1,3-5) + ├── interesting orderings: (+1) + ├── scan episodes + │ ├── columns: id:1(int!null) season:2(int) num:3(int) title:4(string) stardate:5(decimal) + │ ├── stats: [rows=79, distinct(1)=79, null(1)=0, distinct(2)=3, null(2)=0] + │ │ histogram(1)= 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 + │ │ <--- 1 --- 2 --- 3 --- 4 --- 5 --- 6 --- 7 --- 8 --- 9 --- 10 --- 11 --- 12 --- 13 --- 14 --- 15 --- 16 --- 17 --- 18 --- 19 --- 20 --- 21 --- 22 --- 23 --- 24 --- 25 --- 26 --- 27 --- 28 --- 29 --- 30 --- 31 --- 32 --- 33 --- 34 --- 35 --- 36 --- 37 --- 38 --- 39 --- 40 --- 41 --- 42 --- 43 --- 44 --- 45 --- 46 --- 47 --- 48 --- 49 --- 50 --- 51 --- 52 --- 53 --- 54 --- 55 --- 56 --- 57 --- 58 --- 59 --- 60 --- 61 --- 62 --- 63 --- 64 --- 65 --- 66 --- 67 --- 68 --- 69 --- 70 --- 71 --- 72 --- 73 --- 74 --- 75 --- 76 --- 77 --- 78 --- 79 + │ ├── cost: 94.82 + │ ├── key: (1) + │ ├── fd: (1)-->(2-5) + │ ├── prune: (1-5) + │ └── interesting orderings: (+1) + └── filters + └── gt [type=bool, outer=(2), constraints=(/2: [/4 - ]; tight)] + ├── variable: season:2 [type=int] + └── const: 3 [type=int] +(31 rows) +~~~ + + + +To include all details used by the optimizer, including statistics, use `OPT, ENV`. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (OPT, ENV) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +~~~ + +~~~ + text +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + https://cockroachdb.github.io/text/decode.html#eJzEVcFum0oUXWe-4opN7PeAgCFOTKQnEULe4xXjCGhaK4rQGCb2NMBYw5A0qSrlI7zs1_lLKhzbJYumXTQyCyTOuefOnTNzL4oCl4RXlJUWOCy95Qyns7NTcBwf7nqaqquaMiECq6bS03VlOtB7Az07xFiDzufjftI3FTyf50TJML-npT5QDVWTYVLTXEBP62kHmnFg6KD3Lb1vGYYMU6aruqGaXYQUBThhPCM8-cRoWSU5LaiAGa5AzAhk5AbXuYA7nNfEArOJJyWe5CR5pNNHPF2pfhbOyiaezQUt6CPhyQ3jhE7L5JY8VK9okBO6duxCbJ_6LpA5rVhGKuigPZqBF8THEIxiCN77voz2KoIrVq7hZ6isixffgoqcQBSHXvDvViYwz7AgcOY63tD2N7gzCqI4tL0gBmnOaYH5gwQXoTe0wzG8c8fQoRnYkdOV0d65PfT8cSuuQzMZnguSoawLGVZLy7BZrYu6JwjZfuyG6901jODkVp3Xk5ym6na3XvC_68QQxXbsRbHnRLB_hQAAvqzezSOlLK-LspIsuNqCK4Jm0ha4llsCTrAgWYKFZIHU3A1FMxVNB_3IMvuWcajqes8wj__WNEvTpJYyo5WgZSqSlNVloz4atNgZrQRLUpYn4mFOmtyN_219iYsVniS4FixJXnB1nm_zai2Cs_vWeiv8q_zbJjwfxJsbYbzqw449KOvizQ3ovX4TduzAqgN33Q277oT1-Hl7G_p_3gbzFzag6_0ThNyPF77tBdAZXcQyuMFlFyLXbyboX3AejoY_fiMf_nNDdz2l4R8wYBSeuSGcjjeYHTknSFEUBVWMCwTLxWK5eFounqAiOUnFupzl4tsGTnG5zb9lN6IbmgvCq_aJtDNuykDfAwAA__-U4RrF +(1 row) +~~~ + +The output of `EXPLAIN (OPT,ENV)` is now a URL with the data encoded in the fragment portion. Opening the URL shows a page with the decoded data. This change makes it easier to share debugging information across different systems without encountering formatting issues. + +Note that the data is processed in the local browser session and is never sent out over the network. Keep in mind that if you are using any browser extensions, they may be able to access the data locally. + +When you visit the URL above you should see the following output in your browser. + +~~~ +-- Version: CockroachDB CCL v20.2.0 (x86_64-apple-darwin19.3.0, built 2020/05/31 16:16:33, go1.13.4) + +-- reorder_joins_limit has the default value: 4 +-- enable_zigzag_join has the default value: on +-- optimizer_foreign_keys has the default value: on + +CREATE TABLE episodes ( + id INT8 NOT NULL, + season INT8 NULL, + num INT8 NULL, + title STRING NULL, + stardate DECIMAL NULL, + CONSTRAINT "primary" PRIMARY KEY (id ASC), + FAMILY "primary" (id, season, num, title, stardate) +); + +ALTER TABLE startrek.public.episodes INJECT STATISTICS '[ + { + "columns": [ + "id" + ], + "created_at": "2020-04-01 17:46:35.112348+00:00", + "distinct_count": 79, + "histo_col_type": "INT8", + "name": "__auto__", + "null_count": 0, + "row_count": 79 + }, + { + "columns": [ + "season" + ], + "created_at": "2020-04-01 17:46:35.112348+00:00", + "distinct_count": 3, + "histo_col_type": "", + "name": "__auto__", + "null_count": 0, + "row_count": 79 + }, + { + "columns": [ + "num" + ], + "created_at": "2020-04-01 17:46:35.112348+00:00", + "distinct_count": 29, + "histo_col_type": "", + "name": "__auto__", + "null_count": 0, + "row_count": 79 + }, + { + "columns": [ + "title" + ], + "created_at": "2020-04-01 17:46:35.112348+00:00", + "distinct_count": 79, + "histo_col_type": "", + "name": "__auto__", + "null_count": 0, + "row_count": 79 + }, + { + "columns": [ + "stardate" + ], + "created_at": "2020-04-01 17:46:35.112348+00:00", + "distinct_count": 76, + "histo_col_type": "", + "name": "__auto__", + "null_count": 4, + "row_count": 79 + } +]'; + +EXPLAIN (OPT, ENV) SELECT * FROM episodes WHERE season > 3 ORDER BY season ASC; +---- +sort + └── select + ├── scan episodes + └── filters + └── season > 3 +~~~ + +### `VEC` option + +The `VEC` option shows details about the [vectorized execution plan](vectorized-execution.html#how-vectorized-execution-works) for the query. + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (VEC) SELECT * FROM episodes WHERE season > 3; +~~~ + +~~~ + text +--------------------------------------- + │ + └ Node 1 + └ *colexec.selGTInt64Int64ConstOp + └ *colexec.colBatchScan +(4 rows) +~~~ + +The output shows the different internal functions that will be used to process each batch of column-oriented data. + +### `DISTSQL` option + +The `DISTSQL` option generates a URL for a physical query plan that provides high level information about how a query will be executed. For details about reading the physical query plan, see [DistSQL Plan Viewer](explain-analyze.html#distsql-plan-viewer). For more information about distributed SQL queries, see the [DistSQL section of our SQL Layer Architecture docs](architecture/sql-layer.html#distsql). + +{{site.data.alerts.callout_info}} +{% include {{ page.version.version }}/sql/physical-plan-url.md %} +{{site.data.alerts.end}} + +For example, the following `EXPLAIN(DISTSQL)` statement generates a physical plan for a simple query against the [TPC-H database](http://www.tpc.org/tpch/) loaded to a 3-node CockroachDB cluster: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (DISTSQL) SELECT l_shipmode, AVG(l_extendedprice) FROM lineitem GROUP BY l_shipmode; +~~~ + +~~~ + automatic | url +-----------+---------------------------------------------- + true | https://cockroachdb.github.io/distsqlplan... +~~~ + +To view the [DistSQL Plan Viewer](explain-analyze.html#distsql-plan-viewer), point your browser to the URL provided: + +EXPLAIN (DISTSQL) + + To include the data types of the input columns in the physical plan, use `EXPLAIN(DISTSQL, TYPES)`: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN (DISTSQL, TYPES) SELECT l_shipmode, AVG(l_extendedprice) FROM lineitem GROUP BY l_shipmode; +~~~ + +~~~ + automatic | url +-----------+---------------------------------------------- + true | https://cockroachdb.github.io/distsqlplan... +~~~ + +To view the [DistSQL Plan Viewer](explain-analyze.html#distsql-plan-viewer), point your browser to the URL provided: + +EXPLAIN (DISTSQL) + + +### Find the indexes and key ranges a query uses + +You can use `EXPLAIN` to understand which indexes and key ranges queries use, which can help you ensure a query isn't performing a full table scan. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE kv (k INT PRIMARY KEY, v INT); +~~~ + +Because column `v` is not indexed, queries filtering on it alone scan the entire table: + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM kv WHERE v BETWEEN 4 AND 5; +~~~ + +~~~ + tree | field | description +-------+-------------+------------------------ + | distributed | true + | vectorized | false + scan | | + | table | kv@primary + | spans | FULL SCAN + | filter | (v >= 4) AND (v <= 5) +(6 rows) +~~~ + +If there were an index on `v`, CockroachDB would be able to avoid scanning the entire table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE INDEX v ON kv (v); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN SELECT * FROM kv WHERE v BETWEEN 4 AND 5; +~~~ + +~~~ + tree | field | description +-------+-------------+-------------- + | distributed | false + | vectorized | false + scan | | + | table | kv@v + | spans | /4-/6 +(5 rows) +~~~ + +Now, only part of the index `v` is getting scanned, specifically the key range starting at (and including) 4 and stopping before 6. Also note that this query plan is not distributed across nodes on the cluster. + +### Find out if a statement is using `SELECT FOR UPDATE` locking + + CockroachDB has support for ordering transactions by controlling concurrent access to one or more rows of a table using locks. This "`SELECT FOR UPDATE` locking" can result in improved performance for contended operations. It applies to the following statements: + +- [`SELECT FOR UPDATE`](select-for-update.html) +- [`UPDATE`](update.html) + +To see whether a SQL query using one of these statements is using this feature, check the output of `EXPLAIN` for a `locking strength` field as shown below. If the `locking strength` field does not appear, then the statement is not using this feature. + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE IF NOT EXISTS kv (k INT PRIMARY KEY, v INT); +UPSERT INTO kv (k, v) VALUES (1, 5), (2, 10), (3, 15); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> EXPLAIN UPDATE kv SET v = 100 WHERE k = 1; +~~~ + +~~~ + tree | field | description +----------------------+------------------+-------------- + | distributed | false + | vectorized | false + count | | + └── update | | + │ | table | kv + │ | set | v + │ | strategy | updater + │ | auto commit | + └── render | | + └── scan | | + | table | kv@primary + | spans | /1-/1/# + | locking strength | for update +(13 rows) +~~~ + +By default, `SELECT FOR UPDATE` locking is enabled for the initial row scan of `UPDATE` statements. To disable it, toggle the [`enable_implicit_select_for_update` session setting](show-vars.html#enable-implicit-select-for-update). + +## See also + +- [`ALTER TABLE`](alter-table.html) +- [`ALTER SEQUENCE`](alter-sequence.html) +- [`BACKUP`](backup.html) +- [`CANCEL JOB`](cancel-job.html) +- [`CREATE DATABASE`](create-database.html) +- [`DROP DATABASE`](drop-database.html) +- [`EXECUTE`](sql-grammar.html#execute_stmt) +- [`EXPLAIN ANALYZE`](explain-analyze.html) +- [`IMPORT`](import.html) +- [Indexes](indexes.html) +- [`INSERT`](insert.html) +- [`PAUSE JOB`](pause-job.html) +- [`RESET`](reset-vars.html) +- [`RESTORE`](restore.html) +- [`RESUME JOB`](resume-job.html) +- [`SELECT`](select-clause.html) +- [Selection Queries](selection-queries.html) +- [`SET`](set-vars.html) +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`SHOW COLUMNS`](show-columns.html) +- [`UPDATE`](update.html) +- [`UPSERT`](upsert.html) diff --git a/v20.2/export.md b/v20.2/export.md new file mode 100644 index 00000000000..2a9b404f031 --- /dev/null +++ b/v20.2/export.md @@ -0,0 +1,157 @@ +--- +title: EXPORT +summary: Export tabular data from a CockroachDB cluster in CSV format. +toc: true +--- + +The `EXPORT` [statement](sql-statements.html) exports tabular data or the results of arbitrary `SELECT` statements to CSV files. + +Using the [CockroachDB distributed execution engine](https://www.cockroachlabs.com/docs/stable/architecture/sql-layer.html#distsql), `EXPORT` parallelizes CSV creation across all nodes in the cluster, making it possible to quickly get large sets of data out of CockroachDB in a format that can be ingested by downstream systems. If you do not need distributed exports, you can use the [non-enterprise feature to export tabular data in CSV format](#non-distributed-export-using-the-sql-shell). + +{{site.data.alerts.callout_danger}} +This is an [enterprise feature](enterprise-licensing.html). Also, it is in **beta** and is currently undergoing continued testing. Please [file a Github issue](file-an-issue.html) with us if you identify a bug. +{{site.data.alerts.end}} + +## Export file location + +You can use remote cloud storage (Amazon S3, Google Cloud Platform, etc.) to store the exported CSV data. Alternatively, you can use an [HTTP server](create-a-file-server.html) accessible from all nodes. + +For simplicity's sake, it's **strongly recommended** to use cloud/remote storage for the data you want to export. Local files are supported; however, they must be accessible identically from all nodes in the cluster. + +## Cancelling export + +After the export has been initiated, you can cancel it with [`CANCEL QUERY`](cancel-query.html). + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/export.html %}
+ +{{site.data.alerts.callout_info}}The EXPORT statement cannot be used within a transaction.{{site.data.alerts.end}} + +## Required privileges + +Only members of the `admin` role can run `EXPORT`. By default, the `root` user belongs to the `admin` role. + +## Parameters + + Parameter | Description +-----------|------------- + `file_location` | Specify the URL of the file location where you want to store the exported CSV data. + `WITH kv_option` | Control your export's behavior with [these options](#export-options). + `select_stmt` | Specify the query whose result you want to export to CSV format. + `table_name` | Specify the name of the table you want to export to CSV format. + +### Export file URL + +URLs for the file directory location you want to export to must use the following format: + +{% include {{ page.version.version }}/misc/external-urls.md %} + +You can specify the base directory where you want to store the exported .csv files. CockroachDB will create several files in the specified directory with programmatically generated names (e.g., n1.1.csv, n1.2.csv, n2.1.csv, ...). + +### Export options + +You can control the [`EXPORT`](export.html) process's behavior using any of the following key-value pairs as a `kv_option`. + +#### `delimiter` + +If not using comma as your column delimiter, you can specify another ASCII character as the delimiter. + +
+ + + + + + + + + + + + + + + + + + +
Required?No
Keydelimiter
ValueThe ASCII character that delimits columns in your rows
ExampleTo use tab-delimited values: WITH delimiter = e'\t'
+ +#### `nullas` + +Convert SQL *NULL* values so they match the specified string. + + + + + + + + + + + + + + + + + + + + +
Required?No
Keynullas
ValueThe string that should be used to represent NULL values. To avoid collisions, it is important to pick nullas values that does not appear in the exported data.
ExampleTo use empty columns as NULL: WITH nullas = ''
+ +## Examples + +### Export a table + +{% include copy-clipboard.html %} +~~~ sql +> EXPORT INTO CSV + 'azure://acme-co/customer-export-data?AZURE_ACCOUNT_KEY=hash&AZURE_ACCOUNT_NAME=acme-co' + WITH delimiter = '|' FROM TABLE bank.customers; +~~~ + +### Export using a `SELECT` statement + +{% include copy-clipboard.html %} +~~~ sql +> EXPORT INTO CSV + 'azure://acme-co/customer-export-data?AZURE_ACCOUNT_KEY=hash&AZURE_ACCOUNT_NAME=acme-co' + FROM SELECT * FROM bank.customers WHERE id >= 100; +~~~ + +### Non-distributed export using the SQL shell + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql -e "SELECT * from bank.customers WHERE id>=100;" --format=csv > my.csv +~~~ + +### View a running export + +View running exports by using [`SHOW QUERIES`](show-queries.html): + +{% include copy-clipboard.html %} +~~~ sql +> SHOW QUERIES; +~~~ + +### Cancel a running export + +Use [`SHOW QUERIES`](show-queries.html) to get a running export's `query_id`, which can be used to [cancel the export](cancel-query.html): + +{% include copy-clipboard.html %} +~~~ sql +> CANCEL QUERY '14dacc1f9a781e3d0000000000000001'; +~~~ + +## Known limitation + +`EXPORT` may fail with an error if the SQL statements are incompatible with DistSQL. In that case, use the [non-enterprise feature to export tabular data in CSV format](#non-distributed-export-using-the-sql-shell). + +## See also + +- [Create a File Server](create-a-file-server.html) diff --git a/v20.2/file-an-issue.md b/v20.2/file-an-issue.md new file mode 100644 index 00000000000..9d8ff5ba925 --- /dev/null +++ b/v20.2/file-an-issue.md @@ -0,0 +1,80 @@ +--- +title: File an Issue +summary: Learn how to file an issue with CockroachDB +toc: false +--- + +If you've tried to [troubleshoot](troubleshooting-overview.html) an issue yourself, have [reached out for help](support-resources.html), and are still stumped, you can file an issue in GitHub. + +## Guidance + +Before filing an issue: + + - Read the details of the relevant documentation. For example, it's important to understand the function of all flags when [starting nodes](cockroach-start.html). + - Review our [troubleshooting documentation](troubleshooting-overview.html). + - Check our [open issues](https://github.com/cockroachdb/cockroach/issues) for existing tickets related to your problem. Specifically, the [known limitation](https://github.com/cockroachdb/cockroach/issues?q=is%3Aopen+is%3Aissue+label%3Aknown-limitation), [UX surprise](https://github.com/cockroachdb/cockroach/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Aux-surprise) and [SQL semantics](https://github.com/cockroachdb/cockroach/issues?utf8=%E2%9C%93&q=is%3Aopen%20is%3Aissue%20label%3Asql-semantics) flags. If you've encountered one of these issues, please leave a comment about your use case. + - Use our [support resources](support-resources.html), including: + - [StackOverflow](http://stackoverflow.com/questions/tagged/cockroachdb) + - [CockroachDB Community Forum](https://forum.cockroachlabs.com) + - [Gitter](https://gitter.im/cockroachdb/cockroach) + +## Filing an Issue + +To file an issue in GitHub, we need the following information: + +1. A summary of the issue. + +2. The steps to reproduce the issue. + +3. The result you expected. + +4. The result that actually occurred. + +5. The first few lines of the log file from each node in the cluster in a timeframe as close as possible to reproducing the issue. On most Unix-based systems running with defaults, you can get this information using the following command: + + {% include copy-clipboard.html %} + ~~~ shell + $ grep -F '[config]' cockroach-data/logs/cockroach.log + ~~~ + {{site.data.alerts.callout_info}}You might need to replace cockroach-data/logs with the location of your logs.{{site.data.alerts.end}} + If the logs are not available, please include the output of `cockroach version` for each node in the cluster. + +### Template + +You can use this as a template for [filing an issue in GitHub](https://github.com/cockroachdb/cockroach/issues/new): + +~~~ + +## Summary + + + +## Steps to reproduce + +1. +2. +3. + +## Expected Result + + + +## Actual Result + + + +## Log files/version + +### Node 1 + + + +### Node 2 + + + +### Node 3 + + + +~~~ diff --git a/v20.2/float.md b/v20.2/float.md new file mode 100644 index 00000000000..8a1df115e63 --- /dev/null +++ b/v20.2/float.md @@ -0,0 +1,106 @@ +--- +title: FLOAT +summary: The FLOAT data type stores inexact, floating-point numbers with up to 17 digits in total and at least one digit to the right of the decimal point. +toc: true +--- + +CockroachDB supports various inexact, floating-point number [data types](data-types.html) with up to 17 digits of decimal precision. + +They are handled internally using the [standard double-precision (64-bit binary-encoded) IEEE754 format](https://en.wikipedia.org/wiki/IEEE_floating_point). + + +## Names and Aliases + +Name | Aliases +-----|-------- +`FLOAT` | None +`REAL` | `FLOAT4` +`DOUBLE PRECISION` | `FLOAT8` + +## Syntax + +A constant value of type `FLOAT` can be entered as a [numeric literal](sql-constants.html#numeric-literals). +For example: `1.414` or `-1234`. + +The special IEEE754 values for positive infinity, negative infinity +and [NaN (Not-a-Number)](https://en.wikipedia.org/wiki/NaN) cannot be +entered using numeric literals directly and must be converted using an +[interpreted literal](sql-constants.html#interpreted-literals) or an +[explicit conversion](scalar-expressions.html#explicit-type-coercions) +from a string literal instead. + +The following values are recognized: + + Syntax | Value +----------------------------------------|------------------------------------------------ + `inf`, `infinity`, `+inf`, `+infinity` | +∞ + `-inf`, `-infinity` | -∞ + `nan` | [NaN (Not-a-Number)](https://en.wikipedia.org/wiki/NaN) + +For example: + +- `FLOAT '+Inf'` +- `'-Inf'::FLOAT` +- `CAST('NaN' AS FLOAT)` + +## Size + +A `FLOAT` column supports values up to 8 bytes in width, but the total storage size is likely to be larger due to CockroachDB metadata. + +## Examples + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE floats (a FLOAT PRIMARY KEY, b REAL, c DOUBLE PRECISION); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW COLUMNS FROM floats; +~~~ + +~~~ ++-------------+------------------+-------------+----------------+-----------------------+-------------+ +| column_name | data_type | is_nullable | column_default | generation_expression | indices | ++-------------+------------------+-------------+----------------+-----------------------+-------------+ +| a | FLOAT | false | NULL | | {"primary"} | +| b | REAL | true | NULL | | {} | +| c | DOUBLE PRECISION | true | NULL | | {} | ++-------------+------------------+-------------+----------------+-----------------------+-------------+ +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO floats VALUES (1.012345678901, 2.01234567890123456789, CAST('+Inf' AS FLOAT)); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM floats; +~~~ + +~~~ ++----------------+--------------------+------+ +| a | b | c | ++----------------+--------------------+------+ +| 1.012345678901 | 2.0123456789012346 | +Inf | ++----------------+--------------------+------+ +(1 row) +# Note that the value in "b" has been limited to 17 digits. +~~~ + +## Supported casting and conversion + +`FLOAT` values can be [cast](data-types.html#data-type-conversions-and-casts) to any of the following data types: + +Type | Details +-----|-------- +`INT` | Truncates decimal precision and requires values to be between -2^63 and 2^63-1 +`DECIMAL` | Causes an error to be reported if the value is NaN or +/- Inf. +`BOOL` | **0** converts to `false`; all other values convert to `true` +`STRING` | -- + +## See also + +[Data Types](data-types.html) diff --git a/v20.2/flyway.md b/v20.2/flyway.md new file mode 100644 index 00000000000..433a2698c65 --- /dev/null +++ b/v20.2/flyway.md @@ -0,0 +1,180 @@ +--- +title: Flyway +summary: Learn how to use Flyway with CockroachDB. +toc: true +--- + +This page walks you through a series of simple database schema changes using Flyway, an open-source schema migration tool. For detailed information about using Flyway, see the [Flyway documentation site](https://flywaydb.org/documentation/). + +## Before You Begin + +Before you begin, do the following: + +1. [Install CockroachDB](install-cockroachdb.html) and [start a secure cluster](secure-a-cluster.html). +1. Download the latest version of the [Flyway comand-line tool](https://flywaydb.org/documentation/commandline/#download-and-installation). CockroachDB is fully compatible with Flyway versions 6.4.2 and greater. + +## Step 1. Configure Flyway connect to CockroachDB + +1. Extract the Flyway TAR file that you downloaded, and change directories to the extracted `flyway-x.x.x` folder. For example: + + {% include copy-clipboard.html %} + ~~~ shell + $ tar -xvf flyway-commandline-6.4.2-macosx-x64.tar.gz + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cd flyway-6.4.2 + ~~~ + +1. Edit the `flyway-x.x.x/conf/flyway.conf` configuration file to specify the correct [connection parameters](connection-parameters.html) for your running, secure cluster. For example: + + {% include copy-clipboard.html %} + ~~~ conf + ... + flyway.url=jdbc:postgresql://localhost:26257/bank?ssl=true&sslmode=require&sslrootcert=certs/ca.crt&sslkey=certs/client.max.key&sslcert=certs/client.max.crt + flyway.user=max + flyway.password=roach + ... + ~~~ + + {{site.data.alerts.callout_info}} + The SSL connection parameters in the connection URL must specify the full path to the certificates that you generated when you [started the secure cluster](secure-a-cluster.html). Also, the user that you specify (e.g., `max`) must also have [admin privileges](grant.html) on the database whose schema you want to change (e.g., `bank`). + {{site.data.alerts.end}} + +## Step 2. Create a schema migration + +Flyway executes SQL statements defined in `.sql` files located in the `flyway-x.x.x/sql` subdirectory. The schema changes defined in these `.sql` files are known as *migrations*. + +1. Create a `.sql` file with a name that follows the [Flyway naming conventions](https://flywaydb.org/documentation/migrations#naming). For example: + + {% include copy-clipboard.html %} + ~~~ shell + $ touch sql/V1__Add_accounts_table.sql + ~~~ + +1. Edit the `.sql` file, adding a [`CREATE TABLE IF NOT EXISTS`](create-table.html) statement for the table that you want to create, and a simple [`INSERT`](insert.html) statement to initialize the table with some data. For example: + + {% include copy-clipboard.html %} + ~~~ sql + /* Create accounts table */ + CREATE TABLE IF NOT EXISTS accounts ( + id INT PRIMARY KEY, + balance INT + ); + + /* Add initial data to accounts table */ + INSERT INTO accounts (id, balance) VALUES (1, 1000), (2, 250); + ~~~ + +## Step 3. Execute a schema migration + +To execute the migration, run the following command from the top of the `flyway-x.x.x` directory: + +{% include copy-clipboard.html %} +~~~ shell +$ ./flyway migrate +~~~ + +You should see output similar to the following: + +~~~ +Database: jdbc:postgresql://localhost:26257/bank (PostgreSQL 9.5) +Successfully validated 1 migration (execution time 00:00.011s) +Creating Schema History table "bank"."flyway_schema_history" ... +Current version of schema "bank": << Empty Schema >> +Migrating schema "bank" to version 1 - Add accounts table [non-transactional] +Successfully applied 1 migration to schema "bank" (execution time 00:00.081s) +~~~ + +The schema `"bank"` is now on version 1. + +## Step 4. Add additional migrations + +Suppose that you want to change the primary key of the `accounts` table from a simple, incrementing [integer](int.html) (in this case, `id`) to an auto-generated [UUID](uuid.html), to follow some [CockroachDB best practices](performance-best-practices-overview.html#unique-id-best-practices). You can make these changes to the schema by creating and executing an additional migration: + +1. Create a second `.sql` schema migration file, and name the file following the [Flyway naming conventions](https://flywaydb.org/documentation/migrations#naming), to specify a new migration version. For example: + + {% include copy-clipboard.html %} + ~~~ shell + $ touch sql/V2__Alter_accounts_pk.sql + ~~~ + + This file will create a version 2 of the `"bank"` schema. + +1. Edit the `V2__Alter_accounts_pk.sql` migration file, adding some SQL statements that will add a new column to the `accounts` table, and alter the table's primary key. For example: + + {% include copy-clipboard.html %} + ~~~ sql + /* Add new UUID-typed column */ + ALTER TABLE accounts ADD COLUMN unique_id UUID NOT NULL DEFAULT gen_random_uuid(); + + /* Change primary key */ + ALTER TABLE accounts ALTER PRIMARY KEY USING COLUMNS (unique_id); + ~~~ + +1. Execute the migration by running the `flyway migrate` command from the top of the `flyway-x.x.x` directory: + + {% include copy-clipboard.html %} + ~~~ shell + $ ./flyway migrate + ~~~ + + You should see output similar to the following: + + ~~~ + Flyway Community Edition 6.4.2 by Redgate + Database: jdbc:postgresql://localhost:26257/bank (PostgreSQL 9.5) + Successfully validated 2 migrations (execution time 00:00.016s) + Current version of schema "bank": 1 + Migrating schema "bank" to version 2 - Alter accounts pk [non-transactional] + DB: primary key changes are finalized asynchronously; further schema changes on this table may be restricted until the job completes + Successfully applied 1 migration to schema "bank" (execution time 00:00.508s) + ~~~ + + The schema `"bank"` is now on version 2. + +1. Check the complete and pending Flyway migrations with the `flyway info` command: + + ~~~ shell + $ ./flyway info + ~~~ + + ~~~ + Flyway Community Edition 6.4.2 by Redgate + Database: jdbc:postgresql://localhost:26257/bank (PostgreSQL 9.5) + Schema version: 2 + + +-----------+---------+--------------------+------+---------------------+---------+ + | Category | Version | Description | Type | Installed On | State | + +-----------+---------+--------------------+------+---------------------+---------+ + | Versioned | 1 | Add accounts table | SQL | 2020-05-13 17:16:54 | Success | + | Versioned | 2 | Alter accounts pk | SQL | 2020-05-14 13:27:27 | Success | + +-----------+---------+--------------------+------+---------------------+---------+ + ~~~ + +## Flyway and Transactions + +When used with most databases, [Flyway wraps the statements in a migration within a single transaction](https://flywaydb.org/documentation/migrations#transactions). When used with CockroachDB, Flyway does *not* wrap schema migrations in transactions. [Transaction boundaries](transactions.html) are instead handled by CockroachDB. + +### Transaction retries + +When multiple, concurrent transactions or statements are issued to a single CockroachDB cluster, [transaction contention](performance-best-practices-overview.html#understanding-and-avoiding-transaction-contention) can cause schema migrations to fail. In the event of transaction contention, CockroachDB returns a `40001 SQLSTATE` (i.e., a serialization failure), and Flyway automatically retries the migration. For more information about client-side transaction retries in CockroachDB, see [Transaction Retries](transactions.html#transaction-retries). + +## Report Issues with Flyway and CockroachDB + +If you run into problems, please file an issue on the [Flyway issue tracker](https://github.com/flyway/flyway/issues), including the following details about the environment where you encountered the issue: + +- CockroachDB version ([`cockroach version`](cockroach-version.html)) +- Flyway version +- Operating system +- Steps to reproduce the behavior + +## See Also + ++ [Flyway documentation](https://flywaydb.org/documentation/) ++ [Flyway issue tracker](https://github.com/flyway/flyway/issues) ++ [Client connection parameters](connection-parameters.html) ++ [Third-Party Database Tools](third-party-database-tools.html) ++ [Learn CockroachDB SQL](learn-cockroachdb-sql.html) + diff --git a/v20.2/follower-reads.md b/v20.2/follower-reads.md new file mode 100644 index 00000000000..5b9d6104559 --- /dev/null +++ b/v20.2/follower-reads.md @@ -0,0 +1,113 @@ +--- +title: Follower Reads +summary: To reduce latency for read queries, you can choose to have the closest node serve the request using the follower reads feature. +toc: true +--- + +Follower reads are a mechanism that CockroachDB uses to provide faster reads in situations where you can afford to read data that may be slightly less than current (using [`AS OF SYSTEM TIME`](as-of-system-time.html)). Normally, reads have to be serviced by a replica's [leaseholder](architecture/overview.html#architecture-leaseholder). This can be slow, since the leaseholder may be geographically distant from the gateway node that is issuing the query. + +A follower read is a read taken from the closest [replica](architecture/overview.html#architecture-replica), regardless of the replica's leaseholder status. This can result in much better latency in [geo-distributed, multi-region deployments](topology-patterns.html#multi-region-patterns). + + The shortest interval at which [`AS OF SYSTEM TIME`](as-of-system-time.html) can serve follower reads is 4.8 seconds. In prior versions of CockroachDB, the interval was 48 seconds. + +For instructions showing how to use follower reads to get low latency, historical reads in multi-region deployments, see the [Follower Reads Topology Pattern](topology-follower-reads.html). + +{{site.data.alerts.callout_info}} +This is an [enterprise feature](enterprise-licensing.html). +{{site.data.alerts.end}} + +## How follower reads work + +Each CockroachDB node tracks a property called its "closed timestamp", which means that no new writes can ever be introduced below that timestamp. The closed timestamp advances forward by some target interval behind the current time. If the replica receives a write at a timestamp less than its closed timestamp, it rejects the write. + +With [follower reads enabled](#enable-disable-follower-reads), any replica on a node can serve a read for a key as long as the time at which the operation is performed (i.e., the [`AS OF SYSTEM TIME`](as-of-system-time.html) value) is less or equal to the node's closed timestamp. + +When a gateway node in a cluster with follower reads enabled receives a request to read a key with a sufficiently old [`AS OF SYSTEM TIME`](as-of-system-time.html) value, it forwards the request to the closest node that contains a replica of the data–– whether it be a follower or the leaseholder. + +## When to use follower reads + +As long as your [`SELECT` operations](select-clause.html) can tolerate reading data that was current as of the [follower read timestamp](#run-queries-that-use-follower-reads), follower reads can reduce read latencies and increase throughput. + +You should not use follower reads when your application cannot tolerate reading data that was current as of the [follower read timestamp](#run-queries-that-use-follower-reads), since the results of follower reads may not reflect the latest writes against the tables you are querying. + +In addition, follower reads are "read-only" operations; they cannot be used in any way in read-write transactions. + +## Using follower reads + +### Enable/disable follower reads + +Use [`SET CLUSTER SETTING`](set-cluster-setting.html) to set `kv.closed_timestamp.follower_reads_enabled` to: + +- `true` to enable follower reads _(default)_ +- `false` to disable follower reads + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING kv.closed_timestamp.follower_reads_enabled = false; +~~~ + +If you have follower reads enabled, you may want to [verify that follower reads are happening](#verify-that-follower-reads-are-happening). + +{{site.data.alerts.callout_info}} +If follower reads are enabled, but the time-travel query is not using [`AS OF SYSTEM TIME`](as-of-system-time.html) far enough in the past (as defined by the [follower read timestamp](#run-queries-that-use-follower-reads)), CockroachDB does not perform a follower read. Instead, the read accesses the [leaseholder replica](architecture/overview.html#architecture-leaseholder). This adds network latency if the leaseholder is not the closest replica to the gateway node. +{{site.data.alerts.end}} + +### Verify that follower reads are happening + +To verify that your cluster is performing follower reads: + +1. Make sure that [follower reads are enabled](#enable-disable-follower-reads). +2. Go to the [Custom Chart Debug Page in the Admin UI](admin-ui-custom-chart-debug-page.html) and add the metric `follower_read.success_count` to the time series graph you see there. The number of follower reads performed by your cluster will be shown. + +### Run queries that use follower reads + + + +Any [`SELECT` statement](select-clause.html) with an [`AS OF SYSTEM TIME`](as-of-system-time.html) value at least 4.8 seconds in the past can be a follower read (i.e., served by any replica). + +To simplify this calculation, we've added a convenience function that will always set the [`AS OF SYSTEM TIME`](as-of-system-time.html) value to the minimum required for follower reads, `experimental_follower_read_timestamp()`: + +``` sql +SELECT ... FROM ... AS OF SYSTEM TIME experimental_follower_read_timestamp(); +``` + +### Run read-only transactions that use follower reads + +You can set the [`AS OF SYSTEM TIME`](as-of-system-time.html) value for all operations in a read-only transaction: + +```sql +BEGIN; + +SET TRANSACTION AS OF SYSTEM TIME experimental_follower_read_timestamp(); +SELECT ... +SELECT ... + +COMMIT; +``` + +Note that follower reads are "read-only" operations; they cannot be used in any way in read-write transactions. + +{{site.data.alerts.callout_success}} +Using the [`SET TRANSACTION`](set-transaction.html#use-the-as-of-system-time-option) statement as shown in the example above will make it easier to use the follower reads feature from [drivers and ORMs](install-client-drivers.html). +{{site.data.alerts.end}} + +## Follower reads and long-running writes + +Long-running write transactions will create write intents with a timestamp near when the transaction began. When a follower read encounters a write intent, it will often end up in a "transaction wait queue", waiting for the operation to complete; however, this runs counter to the benefit follower reads provides. + +To counteract this, you can issue all follower reads in explicit transactions set with `HIGH` priority: + +```sql +BEGIN PRIORITY HIGH AS OF SYSTEM TIME experimental_follower_read_timestamp(); +SELECT ... +SELECT ... +COMMIT; +``` + +## See Also + +- [Cluster Settings Overview](cluster-settings.html) +- [Load-Based Splitting](load-based-splitting.html) +- [Network Latency Page](admin-ui-network-latency-page.html) +- [Enterprise Features](enterprise-licensing.html) +- [Follower Reads Topology Pattern](topology-follower-reads.html) diff --git a/v20.2/foreign-key.md b/v20.2/foreign-key.md new file mode 100644 index 00000000000..694ec597d15 --- /dev/null +++ b/v20.2/foreign-key.md @@ -0,0 +1,935 @@ +--- +title: Foreign Key Constraint +summary: The `FOREIGN KEY` constraint specifies a column can contain only values exactly matching existing values from the column it references. +toc: true +--- + +A foreign key is a column (or combination of columns) in a table whose values must match values of a column in some other table. `FOREIGN KEY` constraints enforce [referential integrity](https://en.wikipedia.org/wiki/Referential_integrity), which essentially says that if column value A refers to column value B, then column value B must exist. + +For example, given an `orders` table and a `customers` table, if you create a column `orders.customer_id` that references the `customers.id` primary key: + +- Each value inserted or updated in `orders.customer_id` must exactly match a value in `customers.id`, or be `NULL`. +- Values in `customers.id` that are referenced by `orders.customer_id` cannot be deleted or updated, unless you have [cascading actions](#use-a-foreign-key-constraint-with-cascade). However, values of `customers.id` that are _not_ present in `orders.customer_id` can be deleted or updated. + +{{site.data.alerts.callout_info}} + A single column can have multiple foreign key constraints. For an example, see [Add multiple foreign key constraints to a single column](#add-multiple-foreign-key-constraints-to-a-single-column). +{{site.data.alerts.end}} + +## Details + +### Rules for creating foreign keys + +**Foreign Key Columns** + +- Foreign key columns must use their referenced column's [type](data-types.html). +- A foreign key column cannot be a [computed column](computed-columns.html). +- Foreign key columns must be [indexed](indexes.html). + + If you are adding the `FOREIGN KEY` constraint to an existing table, and the columns you want to constraint are not already indexed, use [`CREATE INDEX`](create-index.html) to index them and only then use the [`ADD CONSTRAINT`](add-constraint.html) statement to add the `FOREIGN KEY` constraint to the columns. + + If you are creating a new table, there are a number of ways that you can meet the indexing requirement: + + - You can create indexes explicitly using the [`INDEX`](create-table.html#create-a-table-with-secondary-and-inverted-indexes) clause of `CREATE TABLE`. + - You can rely on indexes created by the [`PRIMARY KEY`](primary-key.html) or [`UNIQUE`](unique.html) constraints. + - If you add a foreign key constraint to an empty table, and an index on the referencing columns does not already exist, CockroachDB automatically creates one. For an example, see [Add the foreign key constraint with `CASCADE`](add-constraint.html#add-the-foreign-key-constraint-with-cascade). It's important to note that if you later remove the `FOREIGN KEY` constraint, this automatically created index _is not_ removed. + + {{site.data.alerts.callout_success}} + Using the foreign key columns as the prefix of an index's columns also satisfies the requirement for an index. For example, if you create foreign key columns `(A, B)`, an index of columns `(A, B, C)` satisfies the requirement for an index. + {{site.data.alerts.end}} + + {{site.data.alerts.callout_info}} + You can drop the index on foreign key columns if another index exists on the same columns and fulfills the indexing requirement described above. + {{site.data.alerts.end}} + +**Referenced Columns** + +- Referenced columns must contain only unique sets of values. This means the `REFERENCES` clause must use exactly the same columns as a [`UNIQUE`](unique.html) or [`PRIMARY KEY`](primary-key.html) constraint on the referenced table. For example, the clause `REFERENCES tbl (C, D)` requires `tbl` to have either the constraint `UNIQUE (C, D)` or `PRIMARY KEY (C, D)`. +- In the `REFERENCES` clause, if you specify a table but no columns, CockroachDB references the table's primary key. In these cases, the `FOREIGN KEY` constraint and the referenced table's primary key must contain the same number of columns. +- Referenced columns must be [indexed](indexes.html). There are a number of ways to meet this requirement: + + - You can create indexes explicitly using the [`INDEX`](create-table.html#create-a-table-with-secondary-and-inverted-indexes) clause of `CREATE TABLE`. + - You can rely on indexes created by the [`PRIMARY KEY`](primary-key.html) or [`UNIQUE`](unique.html) constraints. + - If an index on the referenced column does not already exist, CockroachDB automatically creates one. It's important to note that if you later remove the `FOREIGN KEY` constraint, this automatically created index _is not_ removed. + + {{site.data.alerts.callout_success}} + Using the referenced columns as the prefix of an index's columns also satisfies the requirement for an index. For example, if you create foreign key that references the columns `(A, B)`, an index of columns `(A, B, C)` satisfies the requirement for an index. + {{site.data.alerts.end}} + + {{site.data.alerts.callout_info}} + You can drop the index on the referenced columns if another index exists on the same columns and fulfills the indexing requirement described above. + {{site.data.alerts.end}} + +### Null values + +Single-column foreign keys accept null values. + +Multiple-column (composite) foreign keys only accept null values in the following scenarios: + +- The write contains null values for all foreign key columns (if `MATCH FULL` is specified). +- The write contains null values for at least one foreign key column (if `MATCH SIMPLE` is specified). + +For more information about composite foreign keys, see the [composite foreign key matching](#composite-foreign-key-matching) section. + +Note that allowing null values in either your foreign key or referenced columns can degrade their referential integrity, since any key with a null value is never checked against the referenced table. To avoid this, you can use a [`NOT NULL` constraint](not-null.html) on foreign keys when [creating your tables](create-table.html). + +{{site.data.alerts.callout_info}} +A `NOT NULL` constraint cannot be added to existing tables. +{{site.data.alerts.end}} + +### Composite foreign key matching + +By default, composite foreign keys are matched using the `MATCH SIMPLE` algorithm (which is the same default as Postgres). `MATCH FULL` is available if specified. You can specify both `MATCH FULL` and `MATCH SIMPLE`. + +All composite key matches defined prior to version 19.1 use the `MATCH SIMPLE` comparison method. If you had a composite foreign key constraint and have just upgraded to version 19.1, then please check that `MATCH SIMPLE` works for your schema and consider replacing that foreign key constraint with a `MATCH FULL` one. + +#### How it works + +For matching purposes, composite foreign keys can be in one of three states: + +- **Valid**: Keys that can be used for matching foreign key relationships. + +- **Invalid**: Keys that will not be used for matching (including for any cascading operations). + +- **Unacceptable**: Keys that cannot be inserted at all (an error is signalled). + +`MATCH SIMPLE` stipulates that: + +- **Valid** keys may not contain any null values. + +- **Invalid** keys contain one or more null values. + +- **Unacceptable** keys do not exist from the point of view of `MATCH SIMPLE`; all composite keys are acceptable. + +`MATCH FULL` stipulates that: + +- **Valid** keys may not contain any null values. + +- **Invalid** keys must have all null values. + +- **Unacceptable** keys have any combination of both null and non-null values. In other words, `MATCH FULL` requires that if any column of a composite key is `NULL`, then all columns of the key must be `NULL`. + +For examples showing how these key matching algorithms work, see [Match composite foreign keys with `MATCH SIMPLE` and `MATCH FULL`](#match-composite-foreign-keys-with-match-simple-and-match-full). + +{{site.data.alerts.callout_info}} +CockroachDB does not support `MATCH PARTIAL`. For more information, see issue [#20305](https://github.com/cockroachdb/cockroach/issues/20305). +{{site.data.alerts.end}} + +### Foreign key actions + +When you set a foreign key constraint, you can control what happens to the constrained column when the column it's referencing (the foreign key) is deleted or updated. + +Parameter | Description +----------|------------ +`ON DELETE NO ACTION` | _Default action._ If there are any existing references to the key being deleted, the transaction will fail at the end of the statement. The key can be updated, depending on the `ON UPDATE` action.

Alias: `ON DELETE RESTRICT` +`ON UPDATE NO ACTION` | _Default action._ If there are any existing references to the key being updated, the transaction will fail at the end of the statement. The key can be deleted, depending on the `ON DELETE` action.

Alias: `ON UPDATE RESTRICT` +`ON DELETE RESTRICT` / `ON UPDATE RESTRICT` | `RESTRICT` and `NO ACTION` are currently equivalent until options for deferring constraint checking are added. To set an existing foreign key action to `RESTRICT`, the foreign key constraint must be dropped and recreated. +`ON DELETE CASCADE` / `ON UPDATE CASCADE` | When a referenced foreign key is deleted or updated, all rows referencing that key are deleted or updated, respectively. If there are other alterations to the row, such as a `SET NULL` or `SET DEFAULT`, the delete will take precedence.

Note that `CASCADE` does not list objects it drops or updates, so it should be used cautiously. +`ON DELETE SET NULL` / `ON UPDATE SET NULL` | When a referenced foreign key is deleted or updated, respectively, the columns of all rows referencing that key will be set to `NULL`. The column must allow `NULL` or this update will fail. +`ON DELETE SET DEFAULT` / `ON UPDATE SET DEFAULT` | When a referenced foreign key is deleted or updated, the columns of all rows referencing that key are set to the default value for that column.

If the default value for the column is null, or if no default value is provided and the column does not have a [`NOT NULL`](not-null.html) constraint, this will have the same effect as `ON DELETE SET NULL` or `ON UPDATE SET NULL`. The default value must still conform with all other constraints, such as `UNIQUE`. + +{{site.data.alerts.callout_info}} + If a foreign key column has multiple constraints that reference the same column, the foreign key action that is specified by the first foreign key takes precedence. For an example, see [Add multiple foreign key constraints to a single column](#add-multiple-foreign-key-constraints-to-a-single-column). +{{site.data.alerts.end}} + +### Performance + +Because the foreign key constraint requires per-row checks on two tables, statements involving foreign key or referenced columns can take longer to execute. You're most likely to notice this with operations like bulk inserts into the table with the foreign keys. For bulk inserts into new tables, use the [`IMPORT`](import.html) statement instead of [`INSERT`](insert.html). + +You can improve the performance of some statements that use foreign keys by also using [`INTERLEAVE IN PARENT`](interleave-in-parent.html), but there are tradeoffs. For more information about the performance implications of interleaved tables (as well as the limitations), see the **Interleave tables** section of [Performance best practices](performance-best-practices-overview.html#interleave-tables). + +## Syntax + +Foreign key constraints can be defined at the [table level](#table-level). However, if you only want the constraint to apply to a single column, it can be applied at the [column level](#column-level). + +{{site.data.alerts.callout_info}} +You can also add the `FOREIGN KEY` constraint to existing tables through [`ADD CONSTRAINT`](add-constraint.html#add-the-foreign-key-constraint-with-cascade). +{{site.data.alerts.end}} + +### Column level + +
{% include {{ page.version.version }}/sql/diagrams/foreign_key_column_level.html %}
+ +| Parameter | Description | +|-----------|-------------| +| `table_name` | The name of the table you're creating. | +| `column_name` | The name of the foreign key column. | +| `column_type` | The foreign key column's [data type](data-types.html). | +| `parent_table` | The name of the table the foreign key references. | +| `ref_column_name` | The name of the column the foreign key references.

If you do not include the `ref_column_name` you want to reference from the `parent_table`, CockroachDB uses the first column of `parent_table`'s primary key. +| `column_constraints` | Any other column-level [constraints](constraints.html) you want to apply to this column. | +| `column_def` | Definitions for any other columns in the table. | +| `table_constraints` | Any table-level [constraints](constraints.html) you want to apply. | + +**Example** + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE IF NOT EXISTS orders ( + id INT PRIMARY KEY, + customer INT NOT NULL REFERENCES customers (id) ON DELETE CASCADE, + orderTotal DECIMAL(9,2), + INDEX (customer) + ); +~~~ +{{site.data.alerts.callout_danger}} +`CASCADE` does not list objects it drops or updates, so it should be used cautiously. +{{site.data.alerts.end}} + +### Table level + +
{% include {{ page.version.version }}/sql/diagrams/foreign_key_table_level.html %}
+ +| Parameter | Description | +|-----------|-------------| +| `table_name` | The name of the table you're creating. | +| `column_def` | Definitions for the table's columns. | +| `name` | The name of the constraint. | +| `fk_column_name` | The name of the foreign key column. | +| `parent_table` | The name of the table the foreign key references. | +| `ref_column_name` | The name of the column the foreign key references.

If you do not include the `column_name` you want to reference from the `parent_table`, CockroachDB uses the first column of `parent_table`'s primary key. +| `table_constraints` | Any other table-level [constraints](constraints.html) you want to apply. | + +**Example** + +{% include copy-clipboard.html %} +~~~ sql +CREATE TABLE packages ( + customer INT, + "order" INT, + id INT, + address STRING(50), + delivered BOOL, + delivery_date DATE, + PRIMARY KEY (customer, "order", id), + CONSTRAINT fk_order FOREIGN KEY (customer, "order") REFERENCES orders + ) INTERLEAVE IN PARENT orders (customer, "order") + ; +~~~ + +## Usage examples + +### Use a foreign key constraint with default actions + +In this example, we'll create a table with a foreign key constraint with the default [actions](#foreign-key-actions) (`ON UPDATE NO ACTION ON DELETE NO ACTION`). + +First, create the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers (id INT PRIMARY KEY, email STRING UNIQUE); +~~~ + +Next, create the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE IF NOT EXISTS orders ( + id INT PRIMARY KEY, + customer INT NOT NULL REFERENCES customers (id), + orderTotal DECIMAL(9,2), + INDEX (customer) + ); +~~~ + +Let's insert a record into each table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers VALUES (1001, 'a@co.tld'), (1234, 'info@cockroachlabs.com'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders VALUES (1, 1002, 29.99); +~~~ +~~~ +pq: foreign key violation: value [1002] not found in customers@primary [id] +~~~ + +The second record insertion returns an error because the customer `1002` doesn't exist in the referenced table. + +Let's insert a record into the referencing table and try to update the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders VALUES (1, 1001, 29.99); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers SET id = 1002 WHERE id = 1001; +~~~ +~~~ +pq: foreign key violation: value(s) [1001] in columns [id] referenced in table "orders" +~~~ + +The update to the referenced table returns an error because `id = 1001` is referenced and the default [foreign key action](#foreign-key-actions) is enabled (`ON UPDATE NO ACTION`). However, `id = 1234` is not referenced and can be updated: + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers SET id = 1111 WHERE id = 1234; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers; +~~~ +~~~ + id | email ++------+------------------------+ + 1001 | a@co.tld + 1111 | info@cockroachlabs.com +(2 rows) +~~~ + +Now let's try to delete a referenced row: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers WHERE id = 1001; +~~~ +~~~ +pq: foreign key violation: value(s) [1001] in columns [id] referenced in table "orders" +~~~ + +Similarly, the deletion returns an error because `id = 1001` is referenced and the default [foreign key action](#foreign-key-actions) is enabled (`ON DELETE NO ACTION`). However, `id = 1111` is not referenced and can be deleted: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers WHERE id = 1111; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers; +~~~ +~~~ + id | email ++------+----------+ + 1001 | a@co.tld +(1 row) +~~~ + +### Use a Foreign Key Constraint with `CASCADE` + +In this example, we'll create a table with a foreign key constraint with the [foreign key actions](#foreign-key-actions) `ON UPDATE CASCADE` and `ON DELETE CASCADE`. + +First, create the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers_2 ( + id INT PRIMARY KEY + ); +~~~ + +Then, create the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE orders_2 ( + id INT PRIMARY KEY, + customer_id INT REFERENCES customers_2(id) ON UPDATE CASCADE ON DELETE CASCADE + ); +~~~ + +Insert a few records into the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers_2 VALUES (1), (2), (3); +~~~ + +Insert some records into the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders_2 VALUES (100,1), (101,2), (102,3), (103,1); +~~~ + +Now, let's update an `id` in the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers_2 SET id = 23 WHERE id = 1; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_2; +~~~ +~~~ + id ++----+ + 2 + 3 + 23 +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_2; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | 23 + 101 | 2 + 102 | 3 + 103 | 23 +(4 rows) +~~~ + +When `id = 1` was updated to `id = 23` in `customers_2`, the update propagated to the referencing table `orders_2`. + +Similarly, a deletion will cascade. Let's delete `id = 23` from `customers_2`: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers_2 WHERE id = 23; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_2; +~~~ +~~~ + id ++----+ + 2 + 3 +(2 rows) +~~~ + +Let's check to make sure the rows in `orders_2` where `customers_id = 23` were also deleted: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_2; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 101 | 2 + 102 | 3 +(2 rows) +~~~ + +### Use a Foreign Key Constraint with `SET NULL` + +In this example, we'll create a table with a foreign key constraint with the [foreign key actions](#foreign-key-actions) `ON UPDATE SET NULL` and `ON DELETE SET NULL`. + +First, create the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers_3 ( + id INT PRIMARY KEY + ); +~~~ + +Then, create the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE orders_3 ( + id INT PRIMARY KEY, + customer_id INT REFERENCES customers_3(id) ON UPDATE SET NULL ON DELETE SET NULL + ); +~~~ + +Insert a few records into the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers_3 VALUES (1), (2), (3); +~~~ + +Insert some records into the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders_3 VALUES (100,1), (101,2), (102,3), (103,1); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_3; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | 1 + 101 | 2 + 102 | 3 + 103 | 1 +(4 rows) +~~~ + +Now, let's update an `id` in the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers_3 SET id = 23 WHERE id = 1; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_3; +~~~ +~~~ + id ++----+ + 2 + 3 + 23 +(3 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_3; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | NULL + 101 | 2 + 102 | 3 + 103 | NULL +(4 rows) +~~~ + +When `id = 1` was updated to `id = 23` in `customers_3`, the referencing `customer_id` was set to `NULL`. + +Similarly, a deletion will set the referencing `customer_id` to `NULL`. Let's delete `id = 2` from `customers_3`: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers_3 WHERE id = 2; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_3; +~~~ +~~~ + id ++----+ + 3 + 23 +(2 rows) +~~~ + +Let's check to make sure the row in `orders_3` where `customers_id = 2` was updated to `NULL`: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_3; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | NULL + 101 | NULL + 102 | 3 + 103 | NULL +(4 rows) +~~~ + +### Use a Foreign Key Constraint with `SET DEFAULT` + +In this example, we'll create a table with a `FOREIGN` constraint with the [foreign key actions](#foreign-key-actions) `ON UPDATE SET DEFAULT` and `ON DELETE SET DEFAULT`. + +First, create the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers_4 ( + id INT PRIMARY KEY + ); +~~~ + +Then, create the referencing table with the `DEFAULT` value for `customer_id` set to `9999`: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE orders_4 ( + id INT PRIMARY KEY, + customer_id INT DEFAULT 9999 REFERENCES customers_4(id) ON UPDATE SET DEFAULT ON DELETE SET DEFAULT + ); +~~~ + +Insert a few records into the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers_4 VALUES (1), (2), (3), (9999); +~~~ + +Insert some records into the referencing table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders_4 VALUES (100,1), (101,2), (102,3), (103,1); +~~~ + + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_4; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | 1 + 101 | 2 + 102 | 3 + 103 | 1 +(4 rows) +~~~ + +Now, let's update an `id` in the referenced table: + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers_4 SET id = 23 WHERE id = 1; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_4; +~~~ +~~~ + id ++------+ + 2 + 3 + 23 + 9999 +(4 rows) +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_4; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | 9999 + 101 | 2 + 102 | 3 + 103 | 9999 +(4 rows) +~~~ + +When `id = 1` was updated to `id = 23` in `customers_4`, the referencing `customer_id` was set to `DEFAULT` (i.e., `9999`). You can see this in the first and last rows of `orders_4`, where `id = 100` and the `customer_id` is now `9999` + +Similarly, a deletion will set the referencing `customer_id` to the `DEFAULT` value. Let's delete `id = 2` from `customers_4`: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers_4 WHERE id = 2; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM customers_4; +~~~ +~~~ + id ++------+ + 3 + 23 + 9999 +(3 rows) +~~~ + +Let's check to make sure the corresponding `customer_id` value to `id = 101`, was updated to the `DEFAULT` value (i.e., `9999`) in `orders_4`: + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_4; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 100 | 9999 + 101 | 9999 + 102 | 3 + 103 | 9999 +(4 rows) +~~~ + +If the default value for the `customer_id` column is not set, and the column does not have a [`NOT NULL`](not-null.html) constraint, `ON UPDATE SET DEFAULT` and `ON DELETE SET DEFAULT` actions set referenced column values to `NULL`. + +For example, let's create a new `customers_5` table and insert some values: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers_5 ( + id INT PRIMARY KEY + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers_5 VALUES (1), (2), (3), (4); +~~~ + +Then we can create a new `orders_5` table that references the `customers_5` table, but with no default value specified for the `ON UPDATE SET DEFAULT` and `ON DELETE SET DEFAULT` actions: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE orders_5 ( + id INT PRIMARY KEY, + customer_id INT REFERENCES customers_5(id) ON UPDATE SET DEFAULT ON DELETE SET DEFAULT + ); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders_5 VALUES (200,1), (201,2), (202,3), (203,4); +~~~ + +Deleting and updating values in the `customers_5` table sets the referenced values in `orders_5` to `NULL`: + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM customers_5 WHERE id = 3; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> UPDATE customers_5 SET id = 0 WHERE id = 1; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SELECT * FROM orders_5; +~~~ +~~~ + id | customer_id ++-----+-------------+ + 200 | NULL + 201 | 2 + 202 | NULL + 203 | 4 +(4 rows) +~~~ + +### Add multiple foreign key constraints to a single column + + You can add more than one foreign key constraint to a single column. + +For example, if you create the following tables: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE customers ( + id INT PRIMARY KEY, + name STRING, + email STRING +); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE orders ( + id INT PRIMARY KEY, + customer_id INT UNIQUE, + item_number INT + ); +~~~ + +You can create a table with a column that references columns in both the `customers` and `orders` tables: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE shipments ( + tracking_number UUID DEFAULT gen_random_uuid() PRIMARY KEY, + carrier STRING, + status STRING, + customer_id INT, + CONSTRAINT fk_customers FOREIGN KEY (customer_id) REFERENCES customers(id), + CONSTRAINT fk_orders FOREIGN KEY (customer_id) REFERENCES orders(customer_id) + ); +~~~ + +Inserts into the `shipments` table must fulfill both foreign key constraints on `customer_id` (`fk_customers` and `fk_customers_2`). + +Let's insert a record into each table: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO customers VALUES (1001, 'Alexa', 'a@co.tld'), (1234, 'Evan', 'info@cockroachlabs.com'); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO orders VALUES (1, 1001, 25), (2, 1234, 15), (3, 2000, 5); +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO shipments (carrier, status, customer_id) VALUES ('USPS', 'Out for delivery', 1001); +~~~ + +The last statement succeeds because `1001` matches a unique `id` value in the `customers` table and a unique `customer_id` value in the `orders` table. If `1001` was in neither of the referenced columns, or in just one of them, the statement would return an error. + +For instance, the following statement fulfills just one of the foreign key constraints and returns an error: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT INTO shipments (carrier, status, customer_id) VALUES ('DHL', 'At facility', 2000); +~~~ + +~~~ +ERROR: insert on table "shipments" violates foreign key constraint "fk_customers" +SQLSTATE: 23503 +DETAIL: Key (customer_id)=(2000) is not present in table "customers". +~~~ + +CockroachDB allows you to add multiple foreign key constraints on the same column, that reference the same column: + +{% include copy-clipboard.html %} +~~~ sql +> ALTER TABLE shipments ADD CONSTRAINT fk_customers_2 FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE CASCADE; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CONSTRAINTS FROM shipments; +~~~ + +~~~ + table_name | constraint_name | constraint_type | details | validated +-------------+-----------------+-----------------+----------------------------------------------------------------------+------------ + shipments | fk_customers | FOREIGN KEY | FOREIGN KEY (customer_id) REFERENCES customers(id) | true + shipments | fk_customers_2 | FOREIGN KEY | FOREIGN KEY (customer_id) REFERENCES customers(id) ON DELETE CASCADE | true + shipments | fk_orders | FOREIGN KEY | FOREIGN KEY (customer_id) REFERENCES orders(customer_id) | true + shipments | primary | PRIMARY KEY | PRIMARY KEY (tracking_number ASC) | true +(4 rows) +~~~ + +There are now two foreign key constraints on `customer_id` that reference the `customers(id)` column (i.e., `fk_customers` and `fk_customers_2`). + +In the event of a `DELETE` or `UPDATE` to the referenced column (`customers(id)`), the action for the first foreign key specified takes precedence. In this case, that will be the default [action](#foreign-key-actions) (`ON UPDATE NO ACTION ON DELETE NO ACTION`) on the first foreign key constraint (`fk_customers`). This means that `DELETE`s on referenced columns will fail, even though the second foreign key constraint (`fk_customer_2`) is defined with the `ON DELETE CASCADE` action. + +{% include copy-clipboard.html %} +~~~ sql +> DELETE FROM orders WHERE customer_id = 1001; +~~~ + +~~~ +ERROR: delete on table "orders" violates foreign key constraint "fk_orders" on table "shipments" +SQLSTATE: 23503 +DETAIL: Key (customer_id)=(1001) is still referenced from table "shipments". +~~~ + +### Match composite foreign keys with `MATCH SIMPLE` and `MATCH FULL` + +The examples in this section show how composite foreign key matching works for both the `MATCH SIMPLE` and `MATCH FULL` algorithms. For a conceptual overview, see [Composite foreign key matching](#composite-foreign-key-matching). + +First, let's create some tables. `parent` is a table with a composite key: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE parent (x INT, y INT, z INT, UNIQUE (x, y, z)); +~~~ + +`full_test` has a foreign key on `parent` that uses the `MATCH FULL` algorithm: + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE full_test ( + x INT, + y INT, + z INT, + FOREIGN KEY (x, y, z) REFERENCES parent (x, y, z) MATCH FULL ON DELETE CASCADE ON UPDATE CASCADE + ); +~~~ + +`simple_test` has a foreign key on `parent` that uses the `MATCH SIMPLE` algorithm (the default): + +{% include copy-clipboard.html %} +~~~ sql +> CREATE TABLE simple_test ( + x INT, + y INT, + z INT, + FOREIGN KEY (x, y, z) REFERENCES parent (x, y, z) ON DELETE CASCADE ON UPDATE CASCADE + ); +~~~ + +Next, we populate `parent` with some values: + +{% include copy-clipboard.html %} +~~~ sql +> INSERT + INTO parent + VALUES (1, 1, 1), + (2, 1, 1), + (1, 2, 1), + (1, 1, 2), + (NULL, NULL, NULL), + (1, NULL, NULL), + (NULL, 1, NULL), + (NULL, NULL, 1), + (1, 1, NULL), + (1, NULL, 1), + (NULL, 1, 1); +~~~ + +Now let's look at some `INSERT` statements to see how the different key matching algorithms work. + +- [MATCH SIMPLE](#match-simple) +- [MATCH FULL](#match-full) + +#### MATCH SIMPLE + +Inserting values into the table using the `MATCH SIMPLE` algorithm (described [above](#composite-foreign-key-matching)) gives the following results: + +| Statement | Can insert? | Throws error? | Notes | +|---------------------------------------------------+-------------+---------------+-------------------------------| +| `INSERT INTO simple_test VALUES (1,1,1)` | Yes | No | References `parent (1,1,1)`. | +| `INSERT INTO simple_test VALUES (NULL,NULL,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (1,NULL,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (NULL,1,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (NULL,NULL,1)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (1,1,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (1,NULL,1)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (NULL,1,1)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (2,2,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO simple_test VALUES (2,2,2)` | No | Yes | No `parent` reference exists. | + +#### MATCH FULL + +Inserting values into the table using the `MATCH FULL` algorithm (described [above](#composite-foreign-key-matching)) gives the following results: + +| Statement | Can insert? | Throws error? | Notes | +|-------------------------------------------------+-------------+---------------+-----------------------------------------------------| +| `INSERT INTO full_test VALUES (1,1,1)` | Yes | No | References `parent(1,1,1)`. | +| `INSERT INTO full_test VALUES (NULL,NULL,NULL)` | Yes | No | Does not reference `parent`. | +| `INSERT INTO full_test VALUES (1,NULL,NULL)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (NULL,1,NULL)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (NULL,NULL,1)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (1,1,NULL)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (1,NULL,1)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (NULL,1,1)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (2,2,NULL)` | No | Yes | Can't mix null and non-null values in `MATCH FULL`. | +| `INSERT INTO full_test VALUES (2,2,2)` | No | Yes | No `parent` reference exists. | + +## See also + +- [Constraints](constraints.html) +- [`DROP CONSTRAINT`](drop-constraint.html) +- [`ADD CONSTRAINT`](add-constraint.html) +- [`CHECK` constraint](check.html) +- [`DEFAULT` constraint](default-value.html) +- [`NOT NULL` constraint](not-null.html) +- [`PRIMARY KEY` constraint](primary-key.html) +- [`UNIQUE` constraint](unique.html) +- [`SHOW CONSTRAINTS`](show-constraints.html) diff --git a/v20.2/frequently-asked-questions.md b/v20.2/frequently-asked-questions.md new file mode 100644 index 00000000000..2ca63eb7b1e --- /dev/null +++ b/v20.2/frequently-asked-questions.md @@ -0,0 +1,191 @@ +--- +title: Frequently Asked Questions +summary: CockroachDB FAQ - What is CockroachDB? How does it work? What makes it different from other databases? +tags: postgres, cassandra, google cloud spanner +toc: true +redirect_from: +- simplified-deployment.html +- strong-consistency.html +- sql.html +- distributed-transactions.html +- automated-scaling-and-repair.html +- high-availability.html +- go-implementation.html +- open-source.html +--- + +## What is CockroachDB? + +CockroachDB is a distributed SQL database built on a transactional and strongly-consistent key-value store. It **scales** horizontally; **survives** disk, machine, rack, and even datacenter failures with minimal latency disruption and no manual intervention; supports **strongly-consistent** ACID transactions; and provides a familiar **SQL** API for structuring, manipulating, and querying data. + +CockroachDB is inspired by Google's [Spanner](http://research.google.com/archive/spanner.html) and [F1](http://research.google.com/pubs/pub38125.html) technologies, and the [source code](https://github.com/cockroachdb/cockroach) is freely available. + +## When is CockroachDB a good choice? + +CockroachDB is well suited for applications that require reliable, available, and correct data, and millisecond response times, regardless of scale. It is built to automatically replicate, rebalance, and recover with minimal configuration and operational overhead. Specific use cases include: + +- Distributed or replicated OLTP +- Multi-datacenter deployments +- Multi-region deployments +- Cloud migrations +- Cloud-native infrastructure initiatives + +CockroachDB returns single-row reads in 2ms or less and single-row writes in 4ms or less, and supports a variety of [SQL and operational tuning practices](performance-tuning.html) for optimizing query performance. However, CockroachDB is not yet suitable for heavy analytics / OLAP. + +## How easy is it to install CockroachDB? + +It's as easy as downloading a binary or running our official Kubernetes configurations or Docker image. There are other simple install methods as well, such as running our Homebrew recipe on OS X or building from source files on both OS X and Linux. + +For more details, see [Install CockroachDB](install-cockroachdb.html). + +## How does CockroachDB scale? + +CockroachDB scales horizontally with minimal operator overhead. You can run it on your local computer, a single server, a corporate development cluster, or a private or public cloud. [Adding capacity](cockroach-start.html) is as easy as pointing a new node at the running cluster. + +At the key-value level, CockroachDB starts off with a single, empty range. As you put data in, this single range eventually reaches a threshold size (512 MiB by default). When that happens, the data splits into two ranges, each covering a contiguous segment of the entire key-value space. This process continues indefinitely; as new data flows in, existing ranges continue to split into new ranges, aiming to keep a relatively small and consistent range size. + +When your cluster spans multiple nodes (physical machines, virtual machines, or containers), newly split ranges are automatically rebalanced to nodes with more capacity. CockroachDB communicates opportunities for rebalancing using a peer-to-peer [gossip protocol](https://en.wikipedia.org/wiki/Gossip_protocol) by which nodes exchange network addresses, store capacity, and other information. + +## How does CockroachDB survive failures? + +CockroachDB is designed to survive software and hardware failures, from server restarts to datacenter outages. This is accomplished without confusing artifacts typical of other distributed systems (e.g., stale reads) using strongly-consistent replication as well as automated repair after failures. + +**Replication** + +CockroachDB replicates your data for availability and guarantees consistency between replicas using the [Raft consensus algorithm](https://raft.github.io/), a popular alternative to [Paxos](http://research.microsoft.com/en-us/um/people/lamport/pubs/paxos-simple.pdf). You can [define the location of replicas](configure-replication-zones.html) in various ways, depending on the types of failures you want to secure against and your network topology. You can locate replicas on: + +- Different servers within a rack to tolerate server failures +- Different servers on different racks within a datacenter to tolerate rack power/network failures +- Different servers in different datacenters to tolerate large scale network or power outages + +In a CockroachDB cluster spread across multiple geographic regions, the round-trip latency between regions will have a direct effect on your database's performance. In such cases, it is important to think about the latency requirements of each table and then use the appropriate [data topologies](topology-patterns.html) to locate data for optimal performance and resiliency. For a step-by-step demonstration, see [Low Latency Multi-Region Deployment](demo-low-latency-multi-region-deployment.html). + +**Automated Repair** + +For short-term failures, such as a server restart, CockroachDB uses Raft to continue seamlessly as long as a majority of replicas remain available. Raft makes sure that a new “leader” for each group of replicas is elected if the former leader fails, so that transactions can continue and affected replicas can rejoin their group once they’re back online. For longer-term failures, such as a server/rack going down for an extended period of time or a datacenter outage, CockroachDB automatically rebalances replicas from the missing nodes, using the unaffected replicas as sources. Using capacity information from the gossip network, new locations in the cluster are identified and the missing replicas are re-replicated in a distributed fashion using all available nodes and the aggregate disk and network bandwidth of the cluster. + +## How is CockroachDB strongly-consistent? + +CockroachDB guarantees [serializable SQL transactions](demo-serializable.html), the highest isolation level defined by the SQL standard. It does so by combining the Raft consensus algorithm for writes and a custom time-based synchronization algorithms for reads. + +- Stored data is versioned with MVCC, so [reads simply limit their scope to the data visible at the time the read transaction started](architecture/transaction-layer.html#time-and-hybrid-logical-clocks). + +- Writes are serviced using the [Raft consensus algorithm](https://raft.github.io/), a popular alternative to [Paxos](http://research.microsoft.com/en-us/um/people/lamport/pubs/paxos-simple.pdf). A consensus algorithm guarantees that any majority of replicas together always agree on whether an update was committed successfully. Updates (writes) must reach a majority of replicas (2 out of 3 by default) before they are considered committed. + + To ensure that a write transaction does not interfere with read transactions that start after it, CockroachDB also uses a [timestamp cache](architecture/transaction-layer.html#timestamp-cache) which remembers when data was last read by ongoing transactions. + + This ensures that clients always observe serializable consistency with regards to other concurrent transactions. + +## How is CockroachDB both highly available and strongly consistent? + +The [CAP theorem](https://en.wikipedia.org/wiki/CAP_theorem) states that it is impossible for a distributed system to simultaneously provide more than two out of the following three guarantees: + +- Consistency +- Availability +- Partition Tolerance + +CockroachDB is a CP (consistent and partition tolerant) system. This means +that, in the presence of partitions, the system will become unavailable rather than do anything which might cause inconsistent results. For example, writes require acknowledgements from a majority of replicas, and reads require a lease, which can only be transferred to a different node when writes are possible. + +Separately, CockroachDB is also Highly Available, although "available" here means something different than the way it is used in the CAP theorem. In the CAP theorem, availability is a binary property, but for High Availability, we talk about availability as a spectrum (using terms like "five nines" for a system that is available 99.999% of the time). + +Being both CP and HA means that whenever a majority of replicas can talk to each other, they should be able to make progress. For example, if you deploy CockroachDB to three datacenters and the network link to one of them fails, the other two datacenters should be able to operate normally with only a few seconds' disruption. We do this by attempting to detect partitions and failures quickly and efficiently, transferring leadership to nodes that are able to communicate with the majority, and routing internal traffic away from nodes that are partitioned away. + +## Why is CockroachDB SQL? + +At the lowest level, CockroachDB is a distributed, strongly-consistent, transactional key-value store, but the external API is Standard SQL with extensions. This provides developers familiar relational concepts such as schemas, tables, columns, and indexes and the ability to structure, manipulate, and query data using well-established and time-proven tools and processes. Also, since CockroachDB supports the PostgreSQL wire protocol, it’s simple to get your application talking to Cockroach; just find your [PostgreSQL language-specific driver](install-client-drivers.html) and start building. + +For more details, learn our [basic CockroachDB SQL statements](learn-cockroachdb-sql.html), explore the [full SQL grammar](sql-grammar.html), and try it out via our [built-in SQL client](cockroach-sql.html). Also, to understand how CockroachDB maps SQL table data to key-value storage and how CockroachDB chooses the best index for running a query, see [SQL in CockroachDB](https://www.cockroachlabs.com/blog/sql-in-cockroachdb-mapping-table-data-to-key-value-storage/) and [Index Selection in CockroachDB](https://www.cockroachlabs.com/blog/index-selection-cockroachdb-2/). + +## Does CockroachDB support distributed transactions? + +Yes. CockroachDB distributes transactions across your cluster, whether it’s a few servers in a single location or many servers across multiple datacenters. Unlike with sharded setups, you do not need to know the precise location of data; you just talk to any node in your cluster and CockroachDB gets your transaction to the right place seamlessly. Distributed transactions proceed without downtime or additional latency while rebalancing is underway. You can even move tables – or entire databases – between data centers or cloud infrastructure providers while the cluster is under load. + +## Do transactions in CockroachDB guarantee ACID semantics? + +Yes. Every [transaction](transactions.html) in CockroachDB guarantees [ACID semantics](https://en.wikipedia.org/wiki/ACID) spanning arbitrary tables and rows, even when data is distributed. + +- **Atomicity:** Transactions in CockroachDB are “all or nothing.” If any part of a transaction fails, the entire transaction is aborted, and the database is left unchanged. If a transaction succeeds, all mutations are applied together with virtual simultaneity. For a detailed discussion of atomicity in CockroachDB transactions, see [How CockroachDB Distributes Atomic Transactions](https://www.cockroachlabs.com/blog/how-cockroachdb-distributes-atomic-transactions/). +- **Consistency:** SQL operations never see any intermediate states and move the database from one valid state to another, keeping indexes up to date. Operations always see the results of previously completed statements on overlapping data and maintain specified constraints such as unique columns. For a detailed look at how we've tested CockroachDB for correctness and consistency, see [CockroachDB Beta Passes Jepsen Testing](https://www.cockroachlabs.com/blog/cockroachdb-beta-passes-jepsen-testing/). +- **Isolation:** Transactions in CockroachDB implement the strongest ANSI isolation level: serializable (`SERIALIZABLE`). This means that transactions will never result in anomalies. For more information about transaction isolation in CockroachDB, see [Transactions: Isolation Levels](transactions.html#isolation-levels). +- **Durability:** In CockroachDB, every acknowledged write has been persisted consistently on a majority of replicas (by default, at least 2) via the [Raft consensus algorithm](https://raft.github.io/). Power or disk failures that affect only a minority of replicas (typically 1) do not prevent the cluster from operating and do not lose any data. + +## Since CockroachDB is inspired by Spanner, does it require atomic clocks to synchronize time? + +No. CockroachDB was designed to work without atomic clocks or GPS clocks. It’s a database intended to be run on arbitrary collections of nodes, from physical servers in a corp development cluster to public cloud infrastructure using the flavor-of-the-month virtualization layer. It’d be a showstopper to require an external dependency on specialized hardware for clock synchronization. However, CockroachDB does require moderate levels of clock synchronization for correctness. If clocks drift past a maximum threshold, nodes will be taken offline. It's therefore highly recommended to run [NTP](http://www.ntp.org/) or other clock synchronization software on each node. + +For more details on how CockroachDB handles unsychronized clocks, see [Clock Synchronization](recommended-production-settings.html#clock-synchronization). And for a broader discussion of clocks, and the differences between clocks in Spanner and CockroachDB, see [Living Without Atomic Clocks](https://www.cockroachlabs.com/blog/living-without-atomic-clocks/). + +## What languages can I use to work with CockroachDB? + +CockroachDB supports the PostgreSQL wire protocol, so you can use any available PostgreSQL client drivers. We've tested it from the following languages: + +- Go +- Python +- Ruby +- Java +- JavaScript (node.js) +- C++/C +- Clojure +- PHP +- Rust + +See [Install Client Drivers](install-client-drivers.html) for more details. + +## Why does CockroachDB use the PostgreSQL wire protocol instead of the MySQL protocol? + +CockroachDB uses the PostgreSQL wire protocol because it is better documented than the MySQL protocol, and because PostgreSQL has a liberal Open Source license, similar to BSD or MIT licenses, whereas MySQL has the more restrictive GNU General Public License. + +Note, however, that the protocol used doesn't significantly impact how easy it is to port applications. Swapping out SQL network drivers is rather straightforward in nearly every language. What makes it hard to move from one database to another is the dialect of SQL in use. CockroachDB's dialect is based on PostgreSQL as well. + +## What is CockroachDB’s security model? + +You can run a secure or insecure CockroachDB cluster. When secure, client/node and inter-node communication is encrypted, and SSL certificates authenticate the identity of both clients and nodes. When insecure, there's no encryption or authentication. + +Also, CockroachDB supports common SQL privileges on databases and tables. The `root` user has privileges for all databases, while unique users can be granted privileges for specific statements at the database and table-levels. + +For more details, see our [Security Overview](security-overview.html). + +## How does CockroachDB compare to MySQL or PostgreSQL? + +While all of these databases support SQL syntax, CockroachDB is the only one that scales easily (without the manual complexity of sharding), rebalances and repairs itself automatically, and distributes transactions seamlessly across your cluster. + +For more insight, see [CockroachDB in Comparison](cockroachdb-in-comparison.html). + +## How does CockroachDB compare to Cassandra, HBase, MongoDB, or Riak? + +While all of these are distributed databases, only CockroachDB supports distributed transactions and provides strong consistency. Also, these other databases provide custom APIs, whereas CockroachDB offers standard SQL with extensions. + +For more insight, see [CockroachDB in Comparison](cockroachdb-in-comparison.html). + +## Can a PostgreSQL or MySQL application be migrated to CockroachDB? + +Yes. Most users should be able to follow the instructions in [Migrate from Postgres](migrate-from-postgres.html) or [Migrate from MySQL](migrate-from-mysql.html). Due to differences in available features and syntax, some features supported by these databases may require manual effort to port to CockroachDB. Check those pages for details. + +We also fully support [importing your data via CSV](migrate-from-csv.html). + +## Does Cockroach Labs offer a cloud database as a service? + +Yes. The CockroachCloud offering is currently in Limited Availability and accepting customers on a qualified basis. The offering provides a running CockroachDB cluster suitable to your needs, fully managed by Cockroach Labs on GCP or AWS. Benefits include: + +- No provisioning or deployment efforts for you +- Daily full backups and hourly incremental backups of your data +- Upgrades to the latest stable release of CockroachDB +- Monitoring to provide SLA-level support + +For more details, see the [CockroachCloud](../cockroachcloud/stable/) docs. + +## Why did Cockroach Labs change the license for CockroachDB? + +Our past outlook on the right business model relied on a crucial norm in the OSS world: that companies could build a business around a strong open source core product without a much larger technology platform company coming along and offering the same product as a service. + +Recently, however, OSS companies have seen the rise of highly-integrated providers take advantage of their unique position to offer “as-a-service” versions of OSS products, and offer a superior user experience as a consequence of their integrations. We’ve most recently seen it happen with Amazon’s forked version of ElasticSearch. + +To respond to this breed of competitor, we changed our software licensing terms. To learn more about our motivations, see the [Licensing FAQs](licensing-faqs.html) as well as our [blog post](https://www.cockroachlabs.com/blog/oss-relicensing-cockroachdb/) about the license change. + +## Have questions that weren’t answered? + +- [CockroachDB Community Forum](https://forum.cockroachlabs.com): Ask questions, find answers, and help other users. +- [Join us on Gitter](https://gitter.im/cockroachdb/cockroach): This is the most immediate way to connect with CockroachDB engineers. To open Gitter without leaving these docs, click **Chat with Developers** in the lower-right corner of any page. +- [SQL FAQs](sql-faqs.html): Get answers to frequently asked questions about CockroachDB SQL. +- [Operational FAQS](operational-faqs.html): Get answers to frequently asked questions about operating CockroachDB. diff --git a/v20.2/functions-and-operators.md b/v20.2/functions-and-operators.md new file mode 100644 index 00000000000..f7abf08a986 --- /dev/null +++ b/v20.2/functions-and-operators.md @@ -0,0 +1,124 @@ +--- +title: Functions and Operators +summary: CockroachDB supports many built-in functions, aggregate functions, and operators. +toc: true +--- + +CockroachDB supports the following SQL functions and operators for use in [scalar expressions](scalar-expressions.html). + +{{site.data.alerts.callout_success}}In the built-in SQL shell, use \hf [function] to get inline help about a specific function.{{site.data.alerts.end}} + +## Special syntax forms + +The following syntax forms are recognized for compatibility with the +SQL standard and PostgreSQL, but are equivalent to regular built-in +functions: + +{% include {{ page.version.version }}/sql/function-special-forms.md %} + +## Conditional and function-like operators + +The following table lists the operators that look like built-in +functions but have special evaluation rules: + + Operator | Description +----------|------------- + `ANNOTATE_TYPE(...)` | [Explicitly Typed Expression](scalar-expressions.html#explicitly-typed-expressions) + `ARRAY(...)` | [Conversion of Subquery Results to An Array](scalar-expressions.html#conversion-of-subquery-results-to-an-array) + `ARRAY[...]` | [Conversion of Scalar Expressions to An Array](scalar-expressions.html#array-constructors) + `CAST(...)` | [Type Cast](scalar-expressions.html#explicit-type-coercions) + `COALESCE(...)` | [First non-NULL expression with Short Circuit](scalar-expressions.html#coalesce-and-ifnull-expressions) + `EXISTS(...)` | [Existence Test on the Result of Subqueries](scalar-expressions.html#existence-test-on-the-result-of-subqueries) + `IF(...)` | [Conditional Evaluation](scalar-expressions.html#if-expressions) + `IFNULL(...)` | Alias for `COALESCE` restricted to two operands + `NULLIF(...)` | [Return `NULL` conditionally](scalar-expressions.html#nullif-expressions) + `ROW(...)` | [Tuple Constructor](scalar-expressions.html#tuple-constructor) + +## Built-in functions + +{% include {{ page.version.version }}/sql/functions.md %} + +## Aggregate functions + +For examples showing how to use aggregate functions, see [the `SELECT` clause documentation](select-clause.html#aggregate-functions). + +{{site.data.alerts.callout_info}} +Non-commutative aggregate functions are sensitive to the order in which the rows are processed in the surrounding [`SELECT` clause](select-clause.html#aggregate-functions). To specify the order in which input rows are processed, you can add an [`ORDER BY`](query-order.html) clause within the function argument list. For examples, see the [`SELECT` clause](select-clause.html#order-aggregate-function-input-rows-by-column) documentation. +{{site.data.alerts.end}} + +{% include {{ page.version.version }}/sql/aggregates.md %} + + +## Window functions + +{% include {{ page.version.version }}/sql/window_functions.md %} + +## Operators + +The following table lists all CockroachDB operators from highest to lowest precedence, i.e., the order in which they will be evaluated within a statement. Operators with the same precedence are left associative. This means that those operators are grouped together starting from the left and moving right. + +| Order of Precedence | Operator | Name | Operator Arity | +| ------------------- | -------- | ---- | -------------- | +| 1 | `.` | Member field access operator | binary | +| 2 | `::` | [Type cast](scalar-expressions.html#explicit-type-coercions) | binary | +| 3 | `-` | Unary minus | unary (prefix) | +| | `~` | Bitwise not | unary (prefix) | +| 4 | `^` | Exponentiation | binary | +| 5 | `*` | Multiplication | binary | +| | `/` | Division | binary | +| | `//` | Floor division | binary | +| | `%` | Modulo | binary | +| 6 | `+` | Addition | binary | +| | `-` | Subtraction | binary | +| 7 | `<<` | Bitwise left-shift | binary | +| | `>>` | Bitwise right-shift | binary | +| 8 | `&` | Bitwise AND | binary | +| 9 | `#` | Bitwise XOR | binary | +| 10 | | | Bitwise OR | binary | +| 11 | || | Concatenation | binary | +| | `< ANY`, ` SOME`, ` ALL` | [Multi-valued] "less than" comparison | binary | +| | `> ANY`, ` SOME`, ` ALL` | [Multi-valued] "greater than" comparison | binary | +| | `= ANY`, ` SOME`, ` ALL` | [Multi-valued] "equal" comparison | binary | +| | `<= ANY`, ` SOME`, ` ALL` | [Multi-valued] "less than or equal" comparison | binary | +| | `>= ANY`, ` SOME`, ` ALL` | [Multi-valued] "greater than or equal" comparison | binary | +| | `<> ANY` / `!= ANY`, `<> SOME` / `!= SOME`, `<> ALL` / `!= ALL` | [Multi-valued] "not equal" comparison | binary | +| | `[NOT] LIKE ANY`, `[NOT] LIKE SOME`, `[NOT] LIKE ALL` | [Multi-valued] `LIKE` comparison | binary | +| | `[NOT] ILIKE ANY`, `[NOT] ILIKE SOME`, `[NOT] ILIKE ALL` | [Multi-valued] `ILIKE` comparison | binary | +| 12 | `[NOT] BETWEEN` | Value is [not] within the range specified | binary | +| | `[NOT] BETWEEN SYMMETRIC` | Like `[NOT] BETWEEN`, but in non-sorted order. For example, whereas `a BETWEEN b AND c` means `b <= a <= c`, `a BETWEEN SYMMETRIC b AND c` means `(b <= a <= c) OR (c <= a <= b)`. | binary | +| | `[NOT] IN` | Value is [not] in the set of values specified | binary | +| | `[NOT] LIKE` | Matches [or not] LIKE expression, case sensitive | binary | +| | `[NOT] ILIKE` | Matches [or not] LIKE expression, case insensitive | binary | +| | `[NOT] SIMILAR` | Matches [or not] SIMILAR TO regular expression | binary | +| | `~` | Matches regular expression, case sensitive | binary | +| | `!~` | Does not match regular expression, case sensitive | binary | +| | `~*` | Matches regular expression, case insensitive | binary | +| | `!~*` | Does not match regular expression, case insensitive | binary | +| 13 | `=` | Equal | binary | +| | `<` | Less than | binary | +| | `>` | Greater than | binary | +| | `<=` | Less than or equal to | binary | +| | `>=` | Greater than or equal to | binary | +| | `!=`, `<>` | Not equal | binary | +| 14 | `IS [DISTINCT FROM]` | Equal, considering `NULL` as value | binary | +| | `IS NOT [DISTINCT FROM]` | `a IS NOT b` equivalent to `NOT (a IS b)` | binary | +| | `ISNULL`, `IS UNKNOWN` , `NOTNULL`, `IS NOT UNKNOWN` | Equivalent to `IS NULL` / `IS NOT NULL` | unary (postfix) | +| | `IS NAN`, `IS NOT NAN` | [Comparison with the floating-point NaN value](scalar-expressions.html#comparison-with-nan) | unary (postfix) | +| | `IS OF(...)` | Type predicate | unary (postfix) +| 15 | `NOT` | [Logical NOT](scalar-expressions.html#logical-operators) | unary | +| 16 | `AND` | [Logical AND](scalar-expressions.html#logical-operators) | binary | +| 17 | `OR` | [Logical OR](scalar-expressions.html#logical-operators) | binary | + +[Multi-valued]: scalar-expressions.html#multi-valued-comparisons + +### Supported operations + +{% include {{ page.version.version }}/sql/operators.md %} + + diff --git a/v20.2/get-started-with-enterprise-trial.md b/v20.2/get-started-with-enterprise-trial.md new file mode 100644 index 00000000000..332b3e04f92 --- /dev/null +++ b/v20.2/get-started-with-enterprise-trial.md @@ -0,0 +1,55 @@ +--- +title: Enterprise Trial –– Get Started +summary: Check out this page to get started with your CockroachDB Enterprise Trial +toc: true +license: true +--- + +Congratulations on starting your CockroachDB Enterprise Trial! With it, you'll not only get access to CockroachDB's core capabilities like [high availability](frequently-asked-questions.html#how-does-cockroachdb-survive-failures) and [`SERIALIZABLE` isolation](frequently-asked-questions.html#how-is-cockroachdb-strongly-consistent), but also our Enterprise-only features like distributed [`BACKUP`](backup.html) & [`RESTORE`](restore.html), [geo-partitioning](partitioning.html), and [cluster visualization](enable-node-map.html). + +## Install CockroachDB + +If you haven't already, you'll need to [locally install](install-cockroachdb.html), [remotely deploy](manual-deployment.html), or [orchestrate](orchestration.html) CockroachDB. + +## Enable Enterprise features + +As the CockroachDB `root` user, open the [built-in SQL shell](cockroach-sql.html) in insecure or secure mode, as per your CockroachDB setup. In the following example, we assume that CockroachDB is running in insecure mode. + +{% include copy-clipboard.html %} +~~~ shell +$ cockroach sql --insecure +~~~ + +{{site.data.alerts.callout_info}} +If you've secured your deployment, you'll need to [include the flags for your certificates](cockroach-cert.html) instead of the `--insecure` flag. +{{site.data.alerts.end}} + +Now, use the `SET CLUSTER SETTING` command to set the name of your organization and the license key: + +{% include copy-clipboard.html %} +~~~ sql +> SET CLUSTER SETTING cluster.organization = 'Acme Company'; SET CLUSTER SETTING enterprise.license = 'xxxxxxxxxxxx'; +~~~ + +Then verify your organization in response to the following query: + +{% include copy-clipboard.html %} +~~~ sql +> SHOW CLUSTER SETTING cluster.organization; +~~~ + +## Use Enterprise features + +Your cluster now has access to all of CockroachDB's enterprise features for the length of the trial: + +{% include {{ page.version.version }}/misc/enterprise-features.md %} + +## Getting help + +If you or your team need any help during your trial, our engineers are available on [Gitter](https://gitter.im/cockroachdb/cockroach), [our forum](https://forum.cockroachlabs.com/), or [GitHub](https://github.com/cockroachdb/cockroach).

+ +## See also + +- [Enterprise Licensing](enterprise-licensing.html) +- [`SET CLUSTER SETTING`](set-cluster-setting.html) +- [`SHOW CLUSTER SETTING`](show-cluster-setting.html) diff --git a/v20.2/grant-roles.md b/v20.2/grant-roles.md new file mode 100644 index 00000000000..b8a6bf599dd --- /dev/null +++ b/v20.2/grant-roles.md @@ -0,0 +1,91 @@ +--- +title: GRANT <roles> +summary: The GRANT statement grants user privileges for interacting with specific databases and tables. +toc: true +--- + +The `GRANT ` [statement](sql-statements.html) lets you add a [role](authorization.html#create-and-manage-roles) or [user](authorization.html#create-and-manage-users) as a member to a role. + +{{site.data.alerts.callout_info}} + GRANT <roles> is no longer an enterprise feature and is now freely available in the core version of CockroachDB. +{{site.data.alerts.end}} + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/grant_roles.html %}
+ +## Required privileges + +The user granting role membership must be a role admin (i.e., members with the `WITH ADMIN OPTION`) or a member of the `admin` role. + +To grant membership to the `admin` role, the user must have `WITH ADMIN OPTION` on the `admin` role. + +## Considerations + +- Users and roles can be members of roles. +- The `root` user is automatically created as an `admin` role and assigned the `ALL` privilege for new databases. +- All privileges of a role are inherited by all its members. +- Membership loops are not allowed (direct: `A is a member of B is a member of A` or indirect: `A is a member of B is a member of C ... is a member of A`). + +## Parameters + +Parameter | Description +----------|------------ +`role_name` | The name of the role to which you want to add members. To add members to multiple roles, use a comma-separated list of role names. +`user_name` | The name of the [user](authorization.html#create-and-manage-users) or [role](authorization.html#create-and-manage-roles) to whom you want to grant membership. To add multiple members, use a comma-separated list of user and/or role names. +`WITH ADMIN OPTION` | Designate the user as an role admin. Role admins can grant or revoke membership for the specified role. + +## Examples + +### Grant role membership + +{% include copy-clipboard.html %} +~~~ sql +> GRANT design TO ernie; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON ROLE design; +~~~ +~~~ ++--------+---------+---------+ +| role | member | isAdmin | ++--------+---------+---------+ +| design | barkley | false | +| design | ernie | false | +| design | lola | false | +| design | lucky | false | ++--------+---------+---------+ +~~~ + +### Grant the admin option + +{% include copy-clipboard.html %} +~~~ sql +> GRANT design TO ERNIE WITH ADMIN OPTION; +~~~ +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON ROLE design; +~~~ +~~~ ++--------+---------+---------+ +| role | member | isAdmin | ++--------+---------+---------+ +| design | barkley | false | +| design | ernie | true | +| design | lola | false | +| design | lucky | false | ++--------+---------+---------+ +~~~ + +## See also + +- [Authorization](authorization.html) +- [`REVOKE `](revoke-roles.html) +- [`GRANT `](grant.html) +- [`REVOKE `](revoke.html) +- [`SHOW GRANTS`](show-grants.html) +- [`SHOW ROLES`](show-roles.html) +- [Manage Users](authorization.html#create-and-manage-users) diff --git a/v20.2/grant.md b/v20.2/grant.md new file mode 100644 index 00000000000..bbe28254882 --- /dev/null +++ b/v20.2/grant.md @@ -0,0 +1,168 @@ +--- +title: GRANT <privileges> +summary: The GRANT statement grants user privileges for interacting with specific databases and tables. +toc: true +--- + +The `GRANT ` [statement](sql-statements.html) lets you control each [role](authorization.html#create-and-manage-roles) or [user's](authorization.html#create-and-manage-users) SQL [privileges](authorization.html#assign-privileges) for interacting with specific databases and tables. + +For privileges required by specific statements, see the documentation for the respective [SQL statement](sql-statements.html). + +## Synopsis + +
{% include {{ page.version.version }}/sql/diagrams/grant_privileges.html %}
+ +## Required privileges + + The user granting privileges must also have the privilege being granted on the target database or tables. For example, a user granting the `SELECT` privilege on a table to another user must have the `GRANT` and `SELECT` privileges on that table. + +## Supported privileges + +Roles and users can be granted the following privileges. Some privileges are applicable both for databases and tables, while other are applicable only for tables (see **Levels** in the table below). + +- When a role or user is granted privileges for a database, new tables created in the database will inherit the privileges, but the privileges can then be changed. + + {{site.data.alerts.callout_info}} + The user does not get privileges to existing tables in the database. To grant privileges to a user on all existing tables in a database, see [Grant privileges on all tables in a database](#grant-privileges-on-all-tables-in-a-database) + {{site.data.alerts.end}} + +- When a role or user is granted privileges for a table, the privileges are limited to the table. +- The `root` user automatically belongs to the `admin` role and has the `ALL` privilege for new databases. +- For privileges required by specific statements, see the documentation for the respective [SQL statement](sql-statements.html). + +Privilege | Levels +----------|------------ +`ALL` | Database, Table +`CREATE` | Database, Table +`DROP` | Database, Table +`GRANT` | Database, Table +`SELECT` | Table +`INSERT` | Table +`DELETE` | Table +`UPDATE` | Table + `ZONECONFIG` | Database, Table + +## Parameters + +Parameter | Description +----------|------------ +`table_name` | A comma-separated list of table names. Alternately, to grant privileges to all tables, use `*`. `ON TABLE table.*` grants apply to all existing tables in a database but will not affect tables created after the grant. +`database_name` | A comma-separated list of database names.

Privileges granted on databases will be inherited by any new tables created in the databases, but do not affect existing tables in the database. +`user_name` | A comma-separated list of [users](authorization.html#create-and-manage-users) and/or [roles](authorization.html#create-and-manage-roles) to whom you want to grant privileges. + +## Examples + +### Grant privileges on databases + +{% include copy-clipboard.html %} +~~~ sql +> GRANT CREATE ON DATABASE db1, db2 TO maxroach, betsyroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON DATABASE db1, db2; +~~~ + +~~~ ++----------+------------+------------+ +| Database | User | Privileges | ++----------+------------+------------+ +| db1 | betsyroach | CREATE | +| db1 | maxroach | CREATE | +| db1 | root | ALL | +| db2 | betsyroach | CREATE | +| db2 | maxroach | CREATE | +| db2 | root | ALL | ++----------+------------+------------+ +(6 rows) +~~~ + +### Grant privileges on specific tables in a database + +{% include copy-clipboard.html %} +~~~ sql +> GRANT DELETE ON TABLE db1.t1, db1.t2 TO betsyroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON TABLE db1.t1, db1.t2; +~~~ + +~~~ ++-------+------------+------------+ +| Table | User | Privileges | ++-------+------------+------------+ +| t1 | betsyroach | DELETE | +| t1 | root | ALL | +| t2 | betsyroach | DELETE | +| t2 | root | ALL | ++-------+------------+------------+ +(4 rows) +~~~ + +### Grant privileges on all tables in a database + +{% include copy-clipboard.html %} +~~~ sql +> GRANT SELECT ON TABLE db2.* TO henryroach; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON TABLE db2.*; +~~~ + +~~~ ++-------+------------+------------+ +| Table | User | Privileges | ++-------+------------+------------+ +| t1 | henryroach | SELECT | +| t1 | root | ALL | +| t2 | henryroach | SELECT | +| t2 | root | ALL | ++-------+------------+------------+ +(4 rows) +~~~ + +### Make a table readable to every user in the system + +{% include copy-clipboard.html %} +~~~ sql +> GRANT SELECT ON TABLE myTable TO public; +~~~ + +{% include copy-clipboard.html %} +~~~ sql +> SHOW GRANTS ON TABLE myTable; +~~~ + +~~~ + database_name | schema_name | table_name | grantee | privilege_type ++---------------+-------------+------------+---------+----------------+ + defaultdb | public | mytable | admin | ALL + defaultdb | public | mytable | public | SELECT + defaultdb | public | mytable | root | ALL +(3 rows) +~~~ + +### Grant the privilege to manage the replication zones for a database or table + +{% include copy-clipboard.html %} +~~~ sql +> GRANT ZONECONFIG ON TABLE mytable TO myuser; +~~~ + +The user `myuser` can then use the [`CONFIGURE ZONE`](configure-zone.html) statement to to add, modify, reset, or remove replication zones for the table `mytable`. + +## See also + +- [Authorization](authorization.html) +- [`REVOKE `](revoke-roles.html) +- [`GRANT `](grant-roles.html) +- [`REVOKE `](revoke.html) +- [`SHOW GRANTS`](show-grants.html) +- [`SHOW ROLES`](show-roles.html) +- [`CONFIGURE ZONE`](configure-zone.html) +- [Manage Users](authorization.html#create-and-manage-users) diff --git a/v20.2/gssapi_authentication.md b/v20.2/gssapi_authentication.md new file mode 100644 index 00000000000..ce926f915b8 --- /dev/null +++ b/v20.2/gssapi_authentication.md @@ -0,0 +1,246 @@ +--- +title: GSSAPI Authentication (Enterprise) +summary: Learn about the GSSAPI authentication features for secure CockroachDB clusters. +toc: true +--- + +CockroachDB supports the Generic Security Services API (GSSAPI) with Kerberos authentication. + +{{site.data.alerts.callout_info}} +GSSAPI authentication is an [enterprise-only](enterprise-licensing.html) feature. +{{site.data.alerts.end}} + +## Requirements + +- A working Active Directory or Kerberos environment +- A Service Principal +- A GSSAPI-compatible Postgres Client (psql, etc.) +- A client machine with a Kerberos client installed and configured + +## Configuring KDC for CockroachDB + +To use Kerberos authentication with CockroachDB, configure a Kerberos service principal name (SPN) for CockroachDB and generate a valid keytab file with the following specifications: + +- Set the SPN to the name specified by your client driver. For example, if you use the psql client, set SPN to `postgres`. +- Create SPNs for all DNS addresses that a user would use to connect to your CockroachDB cluster (including any TCP load balancers between the user and the CockroachDB node) and ensure that the keytab contains the keys for every SPN you create. + +### Active Directory + +For Active Directory, the client syntax for generating a keytab that maps a service principal to the SPN is as follows: + +{% include copy-clipboard.html %} +~~~ shell +$ ktpass -out {keytab_filename} -princ {Client_SPN}/{NODE/LB_FQDN}@{DOMAIN} -mapUser {Service_Principal}@{DOMAIN} -mapOp set -pType KRB5_NT_PRINCIPAL +rndPass -crypto AES256-SHA1 +~~~ + +Example: + +{% include copy-clipboard.html %} +~~~ shell +$ ktpass -out postgres.keytab -princ postgres/loadbalancer1.cockroach.industries@COCKROACH.INDUSTRIES -mapUser pguser@COCKROACH.INDUSTRIES -mapOp set -pType KRB5_NT_PRINCIPAL +rndPass -crypto AES256-SHA1 +~~~ + +Copy the resulting keytab to the database nodes. If clients are connecting to multiple addresses (more than one load balancer, or clients connecting directly to nodes), you will need to generate a keytab for each client endpoint. You may want to merge your keytabs together for easier management. You can do this using the `ktpass` command, using the following syntax: + +{% include copy-clipboard.html %} +~~~ shell +$ ktpass -out {new_keytab_filename} -in {old_keytab_filename} -princ {Client_SPN}/{NODE/LB_FQDN}@{DOMAIN} -mapUser {Service_Principal}@{DOMAIN} -mapOp add -pType KRB5_NT_PRINCIPAL +rndPass -crypto AES256-SHA1 +~~~ + +Example (adds `loadbalancer2` to the above example): + +{% include copy-clipboard.html %} +~~~ shell +$ ktpass -out postgres_2lb.keytab -in postgres.keytab -princ postgres/loadbalancer2.cockroach.industries@COCKROACH.INDUSTRIES -mapUser pguser@COCKROACH.INDUSTRIES -mapOp add -pType KRB5_NT_PRINCIPAL +rndPass -crypto AES256-SHA1 +~~~ + +### MIT KDC + +In MIT KDC, you can't map a service principal to an SPN with a different username, so you will need to create a service principal that includes the SPN for your client. + +{% include copy-clipboard.html %} +~~~ shell +$ create-user: kadmin.local -q "addprinc {SPN}/{CLIENT_FQDN}@{DOMAIN}" -pw "{initial_password}" +~~~ + +{% include copy-clipboard.html %} +~~~ shell +$ create-keytab: kadmin.local -q "ktadd -k keytab {SPN}/{CLIENT_FQDN}@{DOMAIN}" +~~~ + +Example: + +{% include copy-clipboard.html %} +~~~ shell +$ kadmin.local -q "addprinc postgres/client2.cockroach.industries@COCKROACH.INDUSTRIES" -pw "testing12345!" +$ kadmin.local -q "ktadd -k keytab postgres/client2.cockroach.industries@COCKROACH.INDUSTRIES" +~~~ + +Copy the resulting keytab to the database nodes. If clients are connecting to multiple addresses (more than one load balancer, or clients connecting directly to nodes), you will need to generate a keytab for each client endpoint. You may want to merge your keytabs together for easier management. The `ktutil` command can be used to read multiple keytab files and output them into a single output [here](https://web.mit.edu/kerberos/krb5-devel/doc/admin/admin_commands/ktutil.html). + + +## Configuring the CockroachDB node +1. Copy the keytab file to a location accessible by the `cockroach` binary. + +2. [Create certificates](cockroach-cert.html) for inter-node and `root` user authentication: + + {% include copy-clipboard.html %} + ~~~ shell + $ mkdir certs my-safe-directory + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-ca \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-node \ + localhost \ + $(hostname) \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach cert create-client \ + root \ + --certs-dir=certs \ + --ca-key=my-safe-directory/ca.key + ~~~ + +3. Provide the path to the keytab in the `KRB5_KTNAME` environment variable. + + Example: `export KRB5_KTNAME=/home/cockroach/postgres.keytab` + +4. Start a CockroachDB node: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach start \ + --certs-dir=certs \ + --listen-addr=0.0.0.0 + ~~~ + +5. Connect to CockroachDB as `root` using the `root` client certificate generated above: + + {% include copy-clipboard.html %} + ~~~ shell + $ cockroach sql --certs-dir=certs + ~~~ + +6. [Enable an enterprise license](enterprise-licensing.html#obtain-a-license). + {{site.data.alerts.callout_info}} You need the enterprise license if you want to use the GSSAPI feature. However, if you only want to test that the GSSAPI setup is working, you don't need to enable an enterprise license. {{site.data.alerts.end}} + +7. Enable GSSAPI authentication: + + {% include copy-clipboard.html %} + ~~~ sql + > SET cluster setting server.host_based_authentication.configuration = 'host all all all gss include_realm=0'; + ~~~ + + Setting the `server.host_based_authentication.configuration` [cluster setting](cluster-settings.html) to this particular value makes it mandatory for all non-`root` users to authenticate using GSSAPI. The `root` user is always an exception and remains able to authenticate using a valid client cert or a user password. + + The `include_realm=0` option is required to tell CockroachDB to remove the `@DOMAIN.COM` realm information from the username. We don't support any advanced mapping of GSSAPI usernames to CockroachDB usernames right now. If you want to limit which realms' users can connect, you can also add one or more `krb_realm` parameters to the end of the line as an allowlist, as follows: `host all all all gss include_realm=0 krb_realm=domain.com krb_realm=corp.domain.com` + + The syntax is based on the `pg_hba.conf` standard for PostgreSQL which is documented [here](https://www.postgresql.org/docs/current/auth-pg-hba-conf.html). It can be used to exclude other users from Kerberos authentication. + +8. Create CockroachDB users for every Kerberos user. Ensure the username does not have the `DOMAIN.COM` realm information. For example, if one of your Kerberos users has a username `carl@realm.com`, then you need to create a CockroachDB user with the username `carl`: + + {% include copy-clipboard.html %} + ~~~ sql + > CREATE USER carl; + ~~~ + + Grant privileges to the user: + + {% include copy-clipboard.html %} + ~~~ sql + > GRANT ALL ON DATABASE defaultdb TO carl; + ~~~ + +## Configuring the client + +{{site.data.alerts.callout_info}} +The `cockroach sql` shell does not yet support GSSAPI authentication. You need to use a GSSAPI-compatible Postgres client, such as Postgres's `psql` client. +{{site.data.alerts.end}} + +1. Install and configure your Kerberos client: + + For CentOS/RHEL systems, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ yum install krb5-user + ~~~ + + For Ubuntu/Debian systems, run: + + {% include copy-clipboard.html %} + ~~~ shell + $ apt-get install krb5-user + ~~~ + + Edit the `/etc/krb5.conf` file to include: + + {% include copy-clipboard.html %} + ~~~ + [libdefaults] + default_realm = {REALM} + + [realms] + {REALM} = { + kdc = {fqdn-kdc-server or ad-server} + admin_server = {fqdn-kdc-server or ad-server} + default_domain = {realm-lower-case} + } + ~~~ + + Example: + + {% include copy-clipboard.html %} + ~~~ + + [libdefaults] + default_realm = COCKROACH.INDUSTRIES + + [realms] + COCKROACH.INDUSTRIES = { + kdc = ad.cockroach.industries + admin_server = ad.cockroach.industries + default_domain = cockroach.industries + } + ~~~ + +2. Get a ticket for the db user: + + {% include copy-clipboard.html %} + ~~~ shell + $ kinit carl + ~~~ + +3. Verify if a valid ticket has been generated: + + {% include copy-clipboard.html %} + ~~~ shell + $ klist + ~~~ + +4. Install the Postgres client (for example, postgresql-client-10 Debian package from postgresql.org). +5. Use the `psql` client, which natively supports GSSAPI authentication, to connect to CockroachDB: + + {% include copy-clipboard.html %} + ~~~ shell + $ psql "postgresql://localhost:26257/defaultdb?sslmode=require" -U carl + ~~~ + +4. If you specified an enterprise license earlier, you should now have a Postgres shell in CockroachDB, indicating that the GSSAPI authentication was successful. If you did not specify an enterprise license, you'll see a message like this: `psql: ERROR: use of GSS authentication requires an enterprise license.` If you see this message, GSSAPI authentication is set up correctly. + +## See also + +- [Authentication](authentication.html) +- [Create Security Certificates](cockroach-cert.html) diff --git a/v20.2/hello-world-example-apps.md b/v20.2/hello-world-example-apps.md new file mode 100644 index 00000000000..ae4b7e75a88 --- /dev/null +++ b/v20.2/hello-world-example-apps.md @@ -0,0 +1,40 @@ +--- +title: Hello World Example Apps +summary: Examples that show you how to build a simple "Hello World" application with CockroachDB +tags: golang, python, java +toc: true +redirect_from: build-an-app-with-cockroachdb.html +--- + +The examples in this section show you how to build simple "Hello World" applications using CockroachDB. + +## Apps + +Click the links in the table below to see simple but complete example applications for each supported language and library combination. + +If you are looking to do a specific task such as connect to the database, insert data, or run multi-statement transactions, see [this list of tasks](#tasks). + +{% include {{page.version.version}}/misc/drivers.md %} + +## See also + +Reference information: + +- [Client drivers](install-client-drivers.html) +- [Third-party database tools](third-party-database-tools.html) +- [Connection parameters](connection-parameters.html) +- [Transactions](transactions.html) +- [Performance best practices](performance-best-practices-overview.html) + + + +Specific tasks: + +- [Connect to the Database](connect-to-the-database.html) +- [Insert Data](insert-data.html) +- [Query Data](query-data.html) +- [Update Data](update-data.html) +- [Delete Data](delete-data.html) +- [Make Queries Fast](make-queries-fast.html) +- [Run Multi-Statement Transactions](run-multi-statement-transactions.html) +- [Error Handling and Troubleshooting](error-handling-and-troubleshooting.html) diff --git a/v20.2/import-data.md b/v20.2/import-data.md new file mode 100644 index 00000000000..4ccc1099c5d --- /dev/null +++ b/v20.2/import-data.md @@ -0,0 +1,5 @@ +--- +title: Import Data +summary: Learn how to import data into a CockroachDB cluster. +toc: true +--- diff --git a/v20.2/import-into.md b/v20.2/import-into.md new file mode 100644 index 00000000000..90d7f44091d --- /dev/null +++ b/v20.2/import-into.md @@ -0,0 +1,253 @@ +--- +title: IMPORT INTO +summary: Import CSV data into an existing CockroachDB table. +toc: true +--- + +The `IMPORT INTO` [statement](sql-statements.html) imports CSV and Avro data into an existing table. To create a table, use [`CREATE TABLE`](create-table.html). + +{{site.data.alerts.callout_success}} +`IMPORT INTO` only works for existing tables. For information on how to import data into new tables, see [`IMPORT`](import.html). +{{site.data.alerts.end}} + +{{site.data.alerts.callout_danger}} +`IMPORT INTO` cannot be used within a [transaction](transactions.html) or during a [rolling upgrade](upgrade-cockroach-version.html). +{{site.data.alerts.end}} + +## Required privileges + +Only members of the `admin` role can run `IMPORT INTO`. By default, the `root` user belongs to the `admin` role. + +## Synopsis + +
+ {% include {{ page.version.version }}/sql/diagrams/import_into.html %} +
+ +{{site.data.alerts.callout_info}} +While importing into an existing table, the table is taken offline. +{{site.data.alerts.end}} + +## Parameters + +Parameter | Description +----------|------------ +`table_name` | The name of the table you want to import into. +`column_name` | The table columns you want to import.

Note: Currently, target columns are not enforced. +`file_location` | The [URL](#import-file-urls) of a CSV or Avro file containing the table data. This can be a comma-separated list of URLs. For an example, see [Import into an existing table from multiple CSV files](#import-into-an-existing-table-from-multiple-csv-files) below. +`