Note
Firehose supports database as a source in all AWS Regions except China Regions, AWS GovCloud (US) Regions, and Asia Pacific (Malaysia). This feature is in preview and is subject to change. Do not use it for your production workloads.
Create the following SQL procedure in your database to create watermark table, role for Firehose access to database, provide required permissions for the Firehose role, and create group ownership role and the Firehose role to the group. You can use this procedure for PostgreSQL databases running on EC2.
Note
Some older database versions may not support the string IF NOT
EXISTS
in the CREATE PROCEDURE line.
In such cases, remove IF NOT EXISTS
from the CREATE PROCEDURE and use the rest of the procedure.
CREATE OR REPLACE PROCEDURE setupFirehose(
p_schema_name TEXT,
p_database_name TEXT,
p_watermark_name TEXT,
p_role_name TEXT,
p_role_password TEXT,
p_group_owner_name TEXT
)
LANGUAGE plpgsql
AS $$
BEGIN
-- Use logical decoding
EXECUTE 'ALTER SYSTEM SET wal_level = logical';
-- Create watermark table
EXECUTE 'CREATE TABLE IF NOT EXISTS ' || quote_ident(p_database_name) || '.' || quote_ident(p_schema_name) || '.' || quote_ident(p_watermark_name) || '(id varchar(64) PRIMARY KEY, type varchar(32), data varchar(2048))';
RAISE NOTICE 'Created watermark table: %', p_watermark_name;
-- Create the role with the given password
IF EXISTS (
SELECT FROM pg_catalog.pg_roles
WHERE rolname = p_role_name)
THEN
RAISE NOTICE 'Role % already exists. Skipping creation', p_role_name;
ELSE
EXECUTE 'CREATE ROLE ' || p_role_name || ' WITH LOGIN INHERIT REPLICATION PASSWORD ' || quote_literal(p_role_password);
RAISE NOTICE 'Created role: %', p_role_name;
END IF;
-- Grant required privileges to the role
EXECUTE 'GRANT CREATE ON SCHEMA ' || quote_ident(p_schema_name) || ' TO ' || quote_ident(p_role_name);
EXECUTE 'GRANT CREATE ON DATABASE ' || quote_ident(p_database_name) || ' TO ' || quote_ident(p_role_name);
EXECUTE 'ALTER TABLE ' || quote_ident(p_schema_name) || '.' || quote_ident(p_watermark_name) || ' OWNER TO ' || quote_ident(p_role_name);
-- Create shared ownership role
IF EXISTS (
SELECT FROM pg_catalog.pg_roles
WHERE rolname = p_group_owner_name)
THEN
RAISE NOTICE 'Role % already exists. Skipping creation', p_group_owner_name;
ELSE
EXECUTE 'CREATE ROLE ' || quote_ident(p_group_owner_name);
RAISE NOTICE 'Created role: %', p_group_owner_name;
END IF;
EXECUTE 'GRANT ' || quote_ident(p_group_owner_name) || ' TO ' || quote_ident(p_role_name);
END;
$$;
Usage
Call this procedure using an SQL Client.
CALL setupFirehose(
'public'
,'test_db'
,'watermark'
,'new_role'
,'Test123'
,'group_role'
);