--- Collecting mzbuild images
materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
$ docker pull --quiet materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
docker.io/materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
$ docker pull --quiet materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
docker.io/materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
$ docker pull --quiet materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
docker.io/materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
$ docker pull --quiet materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
docker.io/materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
$ docker pull --quiet materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
docker.io/materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
$ docker pull --quiet materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
docker.io/materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
$ docker pull --quiet materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
docker.io/materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
$ docker pull --quiet materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
docker.io/materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
$ docker pull --quiet materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
docker.io/materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
$ docker pull --quiet materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
docker.io/materialize/prod-base:mzbuild-QEQJNOGMT43J5XAKL5M6QOXVWUYXY66G
==> Acquiring materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
$ docker pull --quiet materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
docker.io/materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
$ docker pull --quiet materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
docker.io/materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
$ docker pull --quiet materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
docker.io/materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
$ docker pull --quiet materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
docker.io/materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
$ docker pull --quiet materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
docker.io/materialize/jobs:mzbuild-2S64Y7OI5CDWBVL6PNM7FJGJISJ66Q7K
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
$ docker pull --quiet materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
docker.io/materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
$ docker pull --quiet materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
docker.io/materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
$ docker pull --quiet materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
docker.io/materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
$ docker pull --quiet materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
docker.io/materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
$ docker pull --quiet materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
docker.io/materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker pull --quiet materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
docker.io/materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker pull --quiet materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
docker.io/materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker pull --quiet materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
docker.io/materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker pull --quiet materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
docker.io/materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker pull --quiet materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
docker.io/materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
==> Acquiring materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
$ docker pull --quiet materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
docker.io/materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
$ docker pull --quiet materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
docker.io/materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
$ docker pull --quiet materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
docker.io/materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
$ docker pull --quiet materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
docker.io/materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
$ docker pull --quiet materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
docker.io/materialize/test-certs:mzbuild-QQT3XC7KPZNJOMJTEC3JKDAWLL5TTNIJ
==> Acquiring materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
$ docker pull --quiet materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
docker.io/materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
$ docker pull --quiet materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
docker.io/materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
$ docker pull --quiet materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
docker.io/materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
$ docker pull --quiet materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
docker.io/materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
$ docker pull --quiet materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
docker.io/materialize/postgres:mzbuild-ORSA3DY6C63F2F7BOQ6LZLLHY27L7CZT
==> Acquiring materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
$ docker pull --quiet materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
docker.io/materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
$ docker pull --quiet materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
docker.io/materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
$ docker pull --quiet materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
docker.io/materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
$ docker pull --quiet materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
docker.io/materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
$ docker pull --quiet materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
docker.io/materialize/sqlsmith:mzbuild-H5G74ASOX7NEKRVLJSH5WTUJ3K3FO6XH
==> Running test case workflow-default
--- Running workflow default
--- Random seed is 1737073366
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 0
$ docker compose up --detach --wait --quiet-pull toxiproxy
toxiproxy Pulling
toxiproxy Pulled
Network parallel-workload_default Creating
Network parallel-workload_default Created
Volume "parallel-workload_mzdata" Creating
Volume "parallel-workload_mzdata" Created
Volume "parallel-workload_mydata" Creating
Volume "parallel-workload_mydata" Created
Volume "parallel-workload_tmp" Creating
Volume "parallel-workload_tmp" Created
Volume "parallel-workload_scratch" Creating
Volume "parallel-workload_scratch" Created
Container parallel-workload-toxiproxy-1 Creating
Container parallel-workload-toxiproxy-1 Created
Container parallel-workload-toxiproxy-1 Starting
Container parallel-workload-toxiproxy-1 Started
Container parallel-workload-toxiproxy-1 Waiting
Container parallel-workload-toxiproxy-1 Healthy
$ docker compose logs --no-color --timestamps --since 1737073367.2694662
$ docker compose up --detach --wait --quiet-pull cockroach postgres mysql zookeeper kafka schema-registry minio materialized
kafka Pulling
mysql Pulling
minio Pulling
cockroach Pulling
zookeeper Pulling
schema-registry Pulling
cockroach Pulled
minio Pulled
zookeeper Pulled
kafka Pulled
mysql Pulled
schema-registry Pulled
Container parallel-workload-zookeeper-1 Creating
Container parallel-workload-mysql-1 Creating
Container parallel-workload-postgres-1 Creating
Container parallel-workload-cockroach-1 Creating
Container parallel-workload-minio-1 Creating
Container parallel-workload-cockroach-1 Created
Container parallel-workload-minio-1 Created
Container parallel-workload-materialized-1 Creating
Container parallel-workload-mysql-1 Created
Container parallel-workload-postgres-1 Created
Container parallel-workload-zookeeper-1 Created
Container parallel-workload-kafka-1 Creating
Container parallel-workload-materialized-1 Created
Container parallel-workload-kafka-1 Created
Container parallel-workload-schema-registry-1 Creating
Container parallel-workload-schema-registry-1 Created
Container parallel-workload-minio-1 Starting
Container parallel-workload-postgres-1 Starting
Container parallel-workload-cockroach-1 Starting
Container parallel-workload-zookeeper-1 Starting
Container parallel-workload-mysql-1 Starting
Container parallel-workload-minio-1 Started
Container parallel-workload-cockroach-1 Started
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-mysql-1 Started
Container parallel-workload-postgres-1 Started
Container parallel-workload-zookeeper-1 Started
Container parallel-workload-zookeeper-1 Waiting
Container parallel-workload-minio-1 Healthy
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-materialized-1 Starting
Container parallel-workload-zookeeper-1 Healthy
Container parallel-workload-kafka-1 Starting
Container parallel-workload-materialized-1 Started
Container parallel-workload-kafka-1 Started
Container parallel-workload-kafka-1 Waiting
Container parallel-workload-kafka-1 Healthy
Container parallel-workload-schema-registry-1 Starting
Container parallel-workload-schema-registry-1 Started
Container parallel-workload-postgres-1 Waiting
Container parallel-workload-materialized-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-mysql-1 Waiting
Container parallel-workload-schema-registry-1 Waiting
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-zookeeper-1 Waiting
Container parallel-workload-kafka-1 Waiting
Container parallel-workload-postgres-1 Healthy
Container parallel-workload-materialized-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-mysql-1 Healthy
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-zookeeper-1 Healthy
Container parallel-workload-kafka-1 Healthy
Container parallel-workload-schema-registry-1 Healthy
$ docker compose logs --no-color --timestamps --since 1737073377.422409
$ docker compose up --detach --wait --quiet-pull mc
mc Pulling
mc Pulled
Container parallel-workload-mc-1 Creating
Container parallel-workload-mc-1 Created
Container parallel-workload-mc-1 Starting
Container parallel-workload-mc-1 Started
Container parallel-workload-mc-1 Waiting
Container parallel-workload-mc-1 Healthy
$ docker compose exec -T mc mc alias set persist http://minio:9000/ minioadmin minioadmin
Added `persist` successfully.
$ docker compose exec -T mc mc version enable persist/persist
persist/persist versioning is enabled
+++ Running with: --seed=1737073366 --threads=16 --runtime=1500 --complexity=ddl --scenario=0dt-deploy (--host=localhost)
Creating connections
Creating relations
worker_0: Select, SelectOne, CopyToS3, CommitRollback, Reconnect, FlipFlags
worker_1: Insert, SelectOne, HttpPost, CommitRollback, Reconnect, SourceInsert, FlipFlags
worker_2: Delete, Update, InsertReturning, Comment, SetCluster, Reconnect, FlipFlags
worker_3: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_4: Select, SelectOne, CopyToS3, CommitRollback, Reconnect, FlipFlags
worker_5: Delete, Update, InsertReturning, Comment, SetCluster, Reconnect, FlipFlags
worker_6: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_7: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_8: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_9: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_10: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_11: Select, SelectOne, CopyToS3, CommitRollback, Reconnect, FlipFlags
worker_12: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_13: CreateIndex, DropIndex, CreateTable, DropTable, CreateView, DropView, CreateRole, DropRole, CreateCluster, DropCluster, SwapCluster, CreateClusterReplica, DropClusterReplica, SetCluster, CreateWebhookSource, DropWebhookSource, CreateKafkaSink, DropKafkaSink, CreateKafkaSource, DropKafkaSource, CreatePostgresSource, DropPostgresSource, GrantPrivileges, RevokePrivileges, Reconnect, CreateDatabase, DropDatabase, CreateSchema, DropSchema, RenameSchema, RenameTable, RenameView, RenameSink, SwapSchema, FlipFlags
worker_14: Select, SelectOne, CopyToS3, CommitRollback, Reconnect, FlipFlags
worker_15: Select, SelectOne, CopyToS3, CommitRollback, Reconnect, FlipFlags
Deploying generation 1 on materialized2
%3|1737073464.461|FAIL|rdkafka#producer-1| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737073432.5410712
$ docker compose up --detach --wait --quiet-pull materialized2
Container parallel-workload-cockroach-1 Running
Container parallel-workload-minio-1 Running
Container parallel-workload-materialized2-1 Creating
Container parallel-workload-materialized2-1 Created
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized2-1 Starting
%3|1737073465.696|FAIL|rdkafka#producer-2| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Container parallel-workload-materialized2-1 Started
Container parallel-workload-materialized2-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Healthy
Container parallel-workload-cockroach-1 Healthy
%3|1737073470.011|FAIL|rdkafka#producer-3| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Container parallel-workload-materialized2-1 Healthy
Awaiting materialized2 deployment status ReadyToPromote for 900sQPS: 001.0 000.3 000.4 000.3 001.2 000.5 000.3 000.1 000.1 000.5 000.1 000.9 000.2 000.2 000.4 001.1 000.0
.%3|1737073472.208|FAIL|rdkafka#producer-6| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table2"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-2')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
....Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table3"
IN CLUSTER "cluster-1"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-3')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
....QPS: 000.6 000.6 000.4 000.1 000.8 000.5 000.1 000.3 000.2 000.2 000.5 001.0 000.1 000.5 000.7 001.0 000.0
...%3|1737073485.370|FAIL|rdkafka#producer-10| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table5"
IN CLUSTER "cluster-1"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-5')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
..%3|1737073488.251|FAIL|rdkafka#producer-13| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
%3|1737073488.322|FAIL|rdkafka#producer-15| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table6"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-6')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
.%3|1737073489.252|FAIL|rdkafka#producer-13| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
$ docker compose exec -T materialized2 curl -s -X POST localhost:6878/api/leader/promote
Reached!
Awaiting materialized2 deployment status IsLeader for 900s.Network error, retrying
.QPS: 000.8 000.8 000.5 000.4 000.7 000.6 000.3 000.4 000.6 000.5 001.0 000.9 000.1 000.3 001.2 000.7 000.0
.... Reached!
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.3 000.0 000.0 000.2 000.0 000.1 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
Deploying generation 2 on materialized
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.1
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737073464.804839
$ docker compose up --detach --wait --quiet-pull materialized
Container parallel-workload-minio-1 Running
Container parallel-workload-cockroach-1 Running
Container parallel-workload-materialized-1 Recreate
Container parallel-workload-materialized-1 Recreated
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized-1 Starting
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
Container parallel-workload-materialized-1 Started
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-materialized-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized-1 Healthy
Awaiting materialized deployment status ReadyToPromote for 900s....QPS: 002.0 058.3 064.5 000.2 000.9 065.8 000.3 000.6 000.1 000.7 000.2 000.9 000.2 000.1 000.0 000.2 000.0
........%3|1737073671.113|FAIL|rdkafka#producer-17| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
.QPS: 006.9 208.3 196.0 000.2 005.0 195.5 000.4 000.0 000.2 000.3 000.3 006.4 000.3 000.4 002.8 002.7 000.0
%3|1737073672.113|FAIL|rdkafka#producer-17| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
..%3|1737073674.179|FAIL|rdkafka#producer-18| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table8"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-8')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
%3|1737073674.545|FAIL|rdkafka#producer-19| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
.%3|1737073675.546|FAIL|rdkafka#producer-19| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
Reached!
$ docker compose exec -T materialized curl -s -X POST localhost:6878/api/leader/promote
Awaiting materialized deployment status IsLeader for 900s...Network error, retrying
..QPS: 004.1 107.8 091.7 000.1 003.5 089.3 000.2 000.2 000.3 000.1 000.2 003.9 000.2 000.2 002.3 001.7 000.0
. Reached!
QPS: 000.0 000.6 000.0 000.1 000.0 000.1 000.0 000.0 000.0 000.0 000.1 000.0 000.1 000.1 000.0 000.0 000.0
QPS: 000.5 000.5 000.3 000.2 000.6 000.3 000.4 000.3 000.2 000.4 000.5 000.6 000.4 000.3 000.4 000.3 000.0
%3|1737073708.375|FAIL|rdkafka#producer-22| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.4 000.8 000.1 000.5 000.4 000.2 000.6 000.3 000.2 000.2 000.4 000.2 000.4 000.5 000.5 000.7 000.0
%3|1737073714.686|FAIL|rdkafka#producer-23| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.5 000.1 000.2 000.7 000.1 000.1 000.2 000.5 000.4 000.5 000.6 000.5 000.4 000.5 000.4 000.3 000.0
%3|1737073730.006|FAIL|rdkafka#producer-25| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
%3|1737073731.006|FAIL|rdkafka#producer-25| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
QPS: 000.6 000.8 000.2 000.8 000.2 000.2 000.2 000.6 000.5 000.1 000.7 000.6 000.5 000.2 000.6 000.6 000.0
%3|1737073733.041|FAIL|rdkafka#producer-26| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.7 000.1 000.4 000.5 000.8 000.3 000.2 000.2 000.5 000.3 000.5 001.1 000.1 000.3 000.6 000.4 000.0
%3|1737073751.282|FAIL|rdkafka#producer-32| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.7 000.8 000.3 000.4 000.6 000.3 000.2 000.5 000.6 000.4 000.3 000.5 000.5 000.8 000.5 000.5 000.0
QPS: 000.9 001.4 000.3 000.5 000.4 000.3 000.7 000.9 000.6 000.7 000.7 000.1 000.7 000.5 000.8 000.2 000.0
%3|1737073763.383|FAIL|rdkafka#producer-33| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.4 002.1 000.4 000.3 000.3 000.3 000.6 000.6 000.5 000.5 000.5 000.2 000.6 000.6 000.8 000.9 000.0
Deploying generation 3 on materialized2
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737073650.9575408
$ docker compose up --detach --wait --quiet-pull materialized2
Container parallel-workload-minio-1 Running
Container parallel-workload-cockroach-1 Running
Container parallel-workload-materialized2-1 Recreate
Container parallel-workload-materialized2-1 Recreated
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized2-1 Starting
Container parallel-workload-materialized2-1 Started
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-materialized2-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized2-1 Healthy
Awaiting materialized2 deployment status ReadyToPromote for 900s.QPS: 000.3 000.7 000.1 000.5 000.4 000.1 000.4 000.4 000.3 000.4 000.5 000.5 000.3 000.4 000.5 000.3 000.1
.........QPS: 000.5 000.8 000.3 000.8 000.6 000.3 000.9 000.6 000.6 000.8 000.8 000.6 000.8 000.9 000.6 000.6 000.0
..... Reached!
$ docker compose exec -T materialized2 curl -s -X POST localhost:6878/api/leader/promote
Awaiting materialized2 deployment status IsLeader for 900s..QPS: 000.7 001.3 000.8 000.7 000.7 000.3 000.7 000.5 000.4 000.7 000.8 000.9 000.8 000.7 000.6 000.4 000.0
..... Reached!
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
Deploying generation 4 on materialized
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.1
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737073773.4988983
$ docker compose up --detach --wait --quiet-pull materialized
Container parallel-workload-minio-1 Running
Container parallel-workload-cockroach-1 Running
Container parallel-workload-materialized-1 Recreate
Container parallel-workload-materialized-1 Recreated
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized-1 Starting
Container parallel-workload-materialized-1 Started
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-materialized-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized-1 Healthy
Awaiting materialized deployment status ReadyToPromote for 900s..QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
%3|1737073922.732|FAIL|rdkafka#producer-40| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-3"."kafka_table29"
IN CLUSTER "cluster-10"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-29')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
........QPS: 001.9 076.2 218.8 001.5 001.2 000.4 001.9 000.8 000.8 002.1 012.9 002.5 001.4 007.2 004.5 001.6 000.0
..%3|1737073934.378|FAIL|rdkafka#producer-44| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
.%3|1737073935.421|FAIL|rdkafka#producer-46| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table32"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-32')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
... Reached!
$ docker compose exec -T materialized curl -s -X POST localhost:6878/api/leader/promote
Awaiting materialized deployment status IsLeader for 900s..QPS: 001.2 122.9 174.6 007.9 001.7 008.1 002.1 008.9 002.3 003.9 010.3 005.0 002.1 011.9 005.7 002.9 000.0
..... Reached!
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.2 000.6 000.4 000.4 000.4 000.3 000.2 000.6 000.5 000.2 000.2 000.6 000.6 000.4 000.3 000.9 000.0
QPS: 000.4 000.5 000.3 000.4 000.1 000.2 000.6 000.5 000.4 000.3 000.5 000.4 000.4 000.5 000.6 000.3 000.0
QPS: 000.6 000.6 000.2 000.6 000.6 000.2 000.3 000.3 000.5 000.5 000.3 000.6 000.8 000.3 000.6 000.7 000.0
QPS: 000.6 001.1 000.2 000.3 000.4 000.1 000.5 000.4 000.5 000.4 000.5 000.7 000.5 000.4 000.5 000.4 000.0
%3|1737073992.087|FAIL|rdkafka#producer-47| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
%3|1737073999.546|FAIL|rdkafka#producer-49| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.3 000.0 000.2 000.5 000.5 000.0 000.4 000.0 000.3 000.3 000.2 000.5 000.3 000.0 000.2 000.2 000.0
QPS: 000.3 000.9 000.2 000.8 000.6 000.4 001.0 000.5 000.7 000.7 000.7 000.6 000.7 000.6 000.7 000.8 000.0
%3|1737074021.666|FAIL|rdkafka#producer-51| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.1 000.9 000.2 000.3 000.5 000.2 000.4 000.5 000.4 000.5 000.4 000.4 000.6 000.4 000.1 000.4 000.0
%3|1737074022.667|FAIL|rdkafka#producer-51| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
%3|1737074023.722|FAIL|rdkafka#producer-52| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Deploying generation 5 on materialized2
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737073913.650153
QPS: 000.2 000.9 000.1 000.3 000.5 000.3 000.4 000.6 000.7 000.6 000.5 000.3 000.4 000.4 000.4 000.6 000.1
$ docker compose up --detach --wait --quiet-pull materialized2
Container parallel-workload-minio-1 Running
Container parallel-workload-cockroach-1 Running
Container parallel-workload-materialized2-1 Recreate
Container parallel-workload-materialized2-1 Recreated
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized2-1 Starting
Container parallel-workload-materialized2-1 Started
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-materialized2-1 Waiting
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-minio-1 Healthy
Container parallel-workload-materialized2-1 Healthy
Awaiting materialized2 deployment status ReadyToPromote for 900s...QPS: 000.3 001.0 000.2 000.6 000.8 000.3 000.7 000.6 000.8 000.4 000.6 000.3 000.6 000.6 000.9 000.5 000.0
......%3|1737074049.792|FAIL|rdkafka#producer-54| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table39"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-39')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
.%3|1737074050.692|FAIL|rdkafka#producer-56| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
.QPS: 001.2 001.4 000.6 001.3 001.8 000.4 001.4 001.2 001.0 001.4 000.8 001.3 001.1 001.3 001.6 000.2 000.0
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table42"
IN CLUSTER "cluster-55"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-42')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
.%3|1737074053.644|FAIL|rdkafka#producer-59| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
.%3|1737074054.644|FAIL|rdkafka#producer-59| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
... Reached!
$ docker compose exec -T materialized2 curl -s -X POST localhost:6878/api/leader/promote
Awaiting materialized2 deployment status IsLeader for 900s...QPS: 000.4 002.2 000.6 000.5 000.6 000.3 000.4 001.1 001.8 001.2 000.4 000.8 000.9 001.4 001.1 000.9 000.0
... Reached!
QPS: 000.0 000.2 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
Deploying generation 6 on materialized
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.1
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
==> Acquiring materialize/ubuntu-base:mzbuild-BROGB3WOXXKRZHNOL7KLPU4TL5TWLKXX
==> Acquiring materialize/materialized-base:mzbuild-2PYZXDBUVHY5IMEW7H5NXCMHXKK67XZ3
==> Acquiring materialize/materialized:mzbuild-W5EV4HUC4DIPKSSZ64KRIMNRNP7PYMCQ
$ docker compose logs --no-color --timestamps --since 1737074032.038527
$ docker compose up --detach --wait --quiet-pull materialized
Container parallel-workload-minio-1 Running
Container parallel-workload-cockroach-1 Running
Container parallel-workload-materialized-1 Recreate
Container parallel-workload-materialized-1 Recreated
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Healthy
Container parallel-workload-cockroach-1 Healthy
Container parallel-workload-materialized-1 Starting
Container parallel-workload-materialized-1 Started
Container parallel-workload-materialized-1 Waiting
Container parallel-workload-minio-1 Waiting
Container parallel-workload-cockroach-1 Waiting
Container parallel-workload-minio-1 Healthy
Container parallel-workload-cockroach-1 Healthy
QPS: 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0 000.0
Container parallel-workload-materialized-1 Healthy
Awaiting materialized deployment status ReadyToPromote for 900s....Query failed: CREATE SECRET pgpass19 AS 'postgres' cannot write in read-only mode
Query failed: CREATE SECRET pgpass21 AS 'postgres' cannot write in read-only mode
.QPS: 000.5 051.7 058.3 000.3 000.4 000.0 000.5 000.1 000.3 000.3 000.3 000.7 000.1 000.3 000.5 000.5 000.0
.Exception in thread worker_0:
Traceback (most recent call last):
File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
+++ [worker_0] Query failed: SELECT "db-pw-1737073366-0"."s-2"."v-186"."key4-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key0-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key2-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key7-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key8-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0", MAX("db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0") OVER (PARTITION BY "db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0" ORDER BY "db-pw-1737073366-0"."s-2"."v-186"."key4-kafka_table0") FROM "db-pw-1737073366-0"."s-2"."v-186" LIMIT 1; HTTP XX000: internal error: internal error: column 2 is not of expected type ColumnType { scalar_type: String, nullable: false }: Int32(0)
self.run()
File "/usr/lib/python3.12/threading.py", line 1010, in run
self._target(*self._args, **self._kwargs)
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/worker.py", line 111, in run
if action.run(self.exe):
^^^^^^^^^^^^^^^^^^^^
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/action.py", line 319, in run
exe.execute(query, explainable=True, http=Http.RANDOM, fetch=True)
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/executor.py", line 219, in execute
raise QueryError(
materialize.data_ingest.query_error.QueryError: ('HTTP XX000: internal error: internal error: column 2 is not of expected type ColumnType { scalar_type: String, nullable: false }: Int32(0)\n', 'SELECT "db-pw-1737073366-0"."s-2"."v-186"."key4-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key0-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key2-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key7-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key8-kafka_table0", "db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0", MAX("db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0") OVER (PARTITION BY "db-pw-1737073366-0"."s-2"."v-186"."key5-kafka_table0" ORDER BY "db-pw-1737073366-0"."s-2"."v-186"."key4-kafka_table0") FROM "db-pw-1737073366-0"."s-2"."v-186" LIMIT 1;')
.+++ [worker_11] Query failed: COPY (SELECT * FROM "db-pw-1737073366-0"."s-2"."v-186") TO 's3://copytos3/648' WITH (AWS CONNECTION = aws_conn, FORMAT = 'parquet'); Datum Int64(0) does not match builder
Exception in thread worker_11:
Traceback (most recent call last):
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/executor.py", line 152, in execute
self.cur.execute(query.encode())
File "/usr/local/lib/python3.12/dist-packages/psycopg/cursor.py", line 97, in execute
raise ex.with_traceback(None)
psycopg.errors.InternalError_: Datum Int64(0) does not match builder
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/lib/python3.12/threading.py", line 1073, in _bootstrap_inner
self.run()
File "/usr/lib/python3.12/threading.py", line 1010, in run
self._target(*self._args, **self._kwargs)
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/worker.py", line 111, in run
if action.run(self.exe):
^^^^^^^^^^^^^^^^^^^^
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/action.py", line 431, in run
exe.execute(query, explainable=False, http=Http.NO, fetch=False)
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/executor.py", line 154, in execute
raise QueryError(str(e), query)
materialize.data_ingest.query_error.QueryError: ('Datum Int64(0) does not match builder', 'COPY (SELECT * FROM "db-pw-1737073366-0"."s-2"."v-186") TO \'s3://copytos3/648\' WITH (AWS CONNECTION = aws_conn, FORMAT = \'parquet\');')
......%3|1737074201.235|FAIL|rdkafka#producer-62| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table43"
IN CLUSTER "cluster-0"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-43')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
%3|1737074201.457|FAIL|rdkafka#producer-64| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table44"
IN CLUSTER "cluster-55"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-44')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
.%3|1737074201.844|FAIL|rdkafka#producer-66| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT)
QPS: 000.3 128.8 226.7 001.0 003.4 001.5 005.0 001.6 004.5 000.9 008.1 000.6 001.4 001.6 001.8 000.9 000.0
sanity_restart disabled by override(), keeping it disabled
==> mzcompose: test case workflow-default failed: materialize.parallel_workload.worker_exception.WorkerFailedException: ^^^ +++ Thread worker_0 failed, exiting
Traceback (most recent call last):
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/mzcompose/composition.py", line 603, in test_case
yield
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/cli/mzcompose.py", line 692, in handle_composition
composition.workflow(
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/mzcompose/composition.py", line 485, in workflow
func(self, parser)
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/test/parallel-workload/mzcompose.py", line 117, in workflow_default
run(
File "/var/lib/buildkite-agent/builds/hetzner-x86-64-dedi-16cpu-64gb-2f0cfd39/materialize/nightly/misc/python/materialize/parallel_workload/parallel_workload.py", line 322, in run
raise WorkerFailedException(
materialize.parallel_workload.worker_exception.WorkerFailedException: ^^^ +++ Thread worker_0 failed, exiting
--- Uploading report for suite 'mzcompose' to Buildkite Test Analytics
mzcompose: error: at least one test case failed
%3|1737074202.458|FAIL|rdkafka#producer-64| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
%3|1737074202.845|FAIL|rdkafka#producer-66| [thrd:localhost:30123/bootstrap]: localhost:30123/bootstrap: Connect to ipv6#[::1]:30123 failed: Connection refused (after 0ms in state CONNECT, 1 identical error(s) suppressed)
.Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-0"."kafka_table47"
IN CLUSTER "cluster-53"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-47')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
..Query failed: CREATE SOURCE "db-pw-1737073366-0"."s-4"."kafka_table46"
IN CLUSTER "cluster-53"
FROM KAFKA CONNECTION materialize.public.kafka_conn (TOPIC 'data-ingest-46')
FORMAT AVRO
USING CONFLUENT SCHEMA REGISTRY CONNECTION materialize.public.csr_conn
ENVELOPE UPSERT cannot write in read-only mode
$ docker compose exec -T materialized curl -s -X POST localhost:6878/api/leader/promote
Reached!
Awaiting materialized deployment status IsLeader for 900s...... Reached!