Cloud Storage Tutorial

Prerequisites

1. Create a regional bucket, where YOUR_BUCKET_NAME is a globally unique bucket name, and REGION is the region in which you plan to deploy your function:

   gcloud storage buckets create gs://YOUR_BUCKET_NAME --location=REGION

2. To use Cloud Storage functions, grant the pubsub.publisher role to the Cloud Storage service account:

   PROJECT_ID=$(gcloud config get-value project)
   PROJECT_NUMBER=$(gcloud projects list --filter="project_id:$PROJECT_ID" --format='value(project_number)')
   
   SERVICE_ACCOUNT=$(gcloud storage service-agent --project=$PROJECT_ID)
   
   gcloud projects add-iam-policy-binding $PROJECT_ID \
     --member serviceAccount:$SERVICE_ACCOUNT \
     --role roles/pubsub.publisher
   

Prepare the application

1. Clone the sample app repository to your local machine:

   Node.js

   git clone https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   Python

   git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   Go

   git clone https://github.com/GoogleCloudPlatform/golang-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   Java

   git clone https://github.com/GoogleCloudPlatform/java-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   C#

   git clone https://github.com/GoogleCloudPlatform/dotnet-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   Ruby

   git clone https://github.com/GoogleCloudPlatform/ruby-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

   PHP

   git clone https://github.com/GoogleCloudPlatform/php-docs-samples.git

   Alternatively, you can download the sample as a zip file and extract it.

2. Change to the directory that contains the Cloud Run functions sample code:

   Node.js

   cd nodejs-docs-samples/functions/v2/helloGCS/

   Python

   cd python-docs-samples/functions/v2/storage/

   Go

   cd golang-samples/functions/functionsv2/hellostorage/

   Java

   cd java-docs-samples/functions/v2/hello-gcs/

   C#

   cd dotnet-docs-samples/functions/helloworld/HelloGcs/

   Ruby

   cd ruby-docs-samples/functions/helloworld/storage/

   PHP

   cd php-docs-samples/functions/helloworld_storage/

Deploy and trigger the function

Cloud Storage functions are based on Pub/Sub notifications from Cloud Storage and support similar event types:

• finalize

• delete

• archive

• metadata update

The following sections describe how to deploy and trigger a function for each of these event types.

Object Finalize

Object finalize events trigger when a "write" of a Cloud Storage Object is successfully finalized. In particular, this means that creating a new object or overwriting an existing object triggers this event. Archive and metadata update operations are ignored by this trigger.

Object Finalize: deploy the function

Take a look at the sample function, which handles Cloud Storage events:

const functions = require('@google-cloud/functions-framework');

// Register a CloudEvent callback with the Functions Framework that will
// be triggered by Cloud Storage.
functions.cloudEvent('helloGCS', cloudEvent => {
  console.log(`Event ID: ${cloudEvent.id}`);
  console.log(`Event Type: ${cloudEvent.type}`);

  const file = cloudEvent.data;
  console.log(`Bucket: ${file.bucket}`);
  console.log(`File: ${file.name}`);
  console.log(`Metageneration: ${file.metageneration}`);
  console.log(`Created: ${file.timeCreated}`);
  console.log(`Updated: ${file.updated}`);
});

from cloudevents.http import CloudEvent

import functions_framework


# Triggered by a change in a storage bucket
@functions_framework.cloud_event
def hello_gcs(cloud_event: CloudEvent) -> tuple:
    """This function is triggered by a change in a storage bucket.

    Args:
        cloud_event: The CloudEvent that triggered this function.
    Returns:
        The event ID, event type, bucket, name, metageneration, and timeCreated.
    """
    data = cloud_event.data

    event_id = cloud_event["id"]
    event_type = cloud_event["type"]

    bucket = data["bucket"]
    name = data["name"]
    metageneration = data["metageneration"]
    timeCreated = data["timeCreated"]
    updated = data["updated"]

    print(f"Event ID: {event_id}")
    print(f"Event type: {event_type}")
    print(f"Bucket: {bucket}")
    print(f"File: {name}")
    print(f"Metageneration: {metageneration}")
    print(f"Created: {timeCreated}")
    print(f"Updated: {updated}")

    return event_id, event_type, bucket, name, metageneration, timeCreated, updated

// Package helloworld provides a set of Cloud Functions samples.
package helloworld

import (
	"context"
	"fmt"
	"log"
	"strings"

	"github.com/GoogleCloudPlatform/functions-framework-go/functions"
	"github.com/cloudevents/sdk-go/v2/event"
	"github.com/googleapis/google-cloudevents-go/cloud/storagedata"
	"google.golang.org/protobuf/encoding/protojson"
)

func init() {
	functions.CloudEvent("HelloStorage", helloStorage)
}

// helloStorage consumes a CloudEvent message and logs details about the changed object.
func helloStorage(ctx context.Context, e event.Event) error {
	log.Printf("Event ID: %s", e.ID())
	log.Printf("Event Type: %s", e.Type())

	var data storagedata.StorageObjectData
	err := protojson.Unmarshal(e.Data(), &data)

	// Unmarshal() returns an unexported error.
	// Parse the error string to determine whether there is an unknown field.
	if err != nil && strings.Contains(err.Error(), "unknown field") {
		log.Println(err.Error())
		options := protojson.UnmarshalOptions{
			DiscardUnknown: true,
		}
		err = options.Unmarshal(e.Data(), &data)
	}
	if err != nil {
		return fmt.Errorf("protojson.Unmarshal: %w", err)
	}

	log.Printf("Bucket: %s", data.GetBucket())
	log.Printf("File: %s", data.GetName())
	log.Printf("Metageneration: %d", data.GetMetageneration())
	log.Printf("Created: %s", data.GetTimeCreated().AsTime())
	log.Printf("Updated: %s", data.GetUpdated().AsTime())
	return nil
}

import com.google.cloud.functions.CloudEventsFunction;
import com.google.events.cloud.storage.v1.StorageObjectData;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import io.cloudevents.CloudEvent;
import java.nio.charset.StandardCharsets;
import java.util.logging.Logger;

public class HelloGcs implements CloudEventsFunction {
  private static final Logger logger = Logger.getLogger(HelloGcs.class.getName());

  @Override
  public void accept(CloudEvent event) throws InvalidProtocolBufferException {
    logger.info("Event: " + event.getId());
    logger.info("Event Type: " + event.getType());

    if (event.getData() == null) {
      logger.warning("No data found in cloud event payload!");
      return;
    }

    String cloudEventData = new String(event.getData().toBytes(), StandardCharsets.UTF_8);
    StorageObjectData.Builder builder = StorageObjectData.newBuilder();

    // If you do not ignore unknown fields, then JsonFormat.Parser returns an
    // error when encountering a new or unknown field. Note that you might lose
    // some event data in the unmarshaling process by ignoring unknown fields.
    JsonFormat.Parser parser = JsonFormat.parser().ignoringUnknownFields();
    parser.merge(cloudEventData, builder);
    StorageObjectData data = builder.build();

    logger.info("Bucket: " + data.getBucket());
    logger.info("File: " + data.getName());
    logger.info("Metageneration: " + data.getMetageneration());
    logger.info("Created: " + data.getTimeCreated());
    logger.info("Updated: " + data.getUpdated());
  }
}

using CloudNative.CloudEvents;
using Google.Cloud.Functions.Framework;
using Google.Events.Protobuf.Cloud.Storage.V1;
using Microsoft.Extensions.Logging;
using System.Threading;
using System.Threading.Tasks;

namespace HelloGcs;

 /// <summary>
 /// Example Cloud Storage-triggered function.
 /// This function can process any event from Cloud Storage.
 /// </summary>
public class Function : ICloudEventFunction<StorageObjectData>
{
    private readonly ILogger _logger;

    public Function(ILogger<Function> logger) =>
        _logger = logger;

    public Task HandleAsync(CloudEvent cloudEvent, StorageObjectData data, CancellationToken cancellationToken)
    {
        _logger.LogInformation("Event: {event}", cloudEvent.Id);
        _logger.LogInformation("Event Type: {type}", cloudEvent.Type);
        _logger.LogInformation("Bucket: {bucket}", data.Bucket);
        _logger.LogInformation("File: {file}", data.Name);
        _logger.LogInformation("Metageneration: {metageneration}", data.Metageneration);
        _logger.LogInformation("Created: {created:s}", data.TimeCreated?.ToDateTimeOffset());
        _logger.LogInformation("Updated: {updated:s}", data.Updated?.ToDateTimeOffset());
        return Task.CompletedTask;
    }
}

require "functions_framework"

FunctionsFramework.cloud_event "hello_gcs" do |event|
  # This function supports all Cloud Storage events.
  # The `event` parameter is a CloudEvents::Event::V1 object.
  # See https://cloudevents.github.io/sdk-ruby/latest/CloudEvents/Event/V1.html
  payload = event.data

  logger.info "Event: #{event.id}"
  logger.info "Event Type: #{event.type}"
  logger.info "Bucket: #{payload['bucket']}"
  logger.info "File: #{payload['name']}"
  logger.info "Metageneration: #{payload['metageneration']}"
  logger.info "Created: #{payload['timeCreated']}"
  logger.info "Updated: #{payload['updated']}"
end

use CloudEvents\V1\CloudEventInterface;
use Google\CloudFunctions\FunctionsFramework;

// Register the function with Functions Framework.
// This enables omitting the `FUNCTIONS_SIGNATURE_TYPE=cloudevent` environment
// variable when deploying. The `FUNCTION_TARGET` environment variable should
// match the first parameter.
FunctionsFramework::cloudEvent('helloGCS', 'helloGCS');

function helloGCS(CloudEventInterface $cloudevent)
{
    // This function supports all Cloud Storage event types.
    $log = fopen(getenv('LOGGER_OUTPUT') ?: 'php://stderr', 'wb');
    $data = $cloudevent->getData();
    fwrite($log, 'Event: ' . $cloudevent->getId() . PHP_EOL);
    fwrite($log, 'Event Type: ' . $cloudevent->getType() . PHP_EOL);
    fwrite($log, 'Bucket: ' . $data['bucket'] . PHP_EOL);
    fwrite($log, 'File: ' . $data['name'] . PHP_EOL);
    fwrite($log, 'Metageneration: ' . $data['metageneration'] . PHP_EOL);
    fwrite($log, 'Created: ' . $data['timeCreated'] . PHP_EOL);
    fwrite($log, 'Updated: ' . $data['updated'] . PHP_EOL);
}

To deploy the function, run the following command in the directory where the sample code is located:

gcloud functions deploy nodejs-finalize-function \
--gen2 \
--runtime=nodejs22 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy python-finalize-function \
--gen2 \
--runtime=python312 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy go-finalize-function \
--gen2 \
--runtime=go122 \
--region=REGION \
--source=. \
--entry-point=HelloStorage \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy java-finalize-function \
--gen2 \
--runtime=java21 \
--region=REGION \
--source=. \
--entry-point=functions.HelloGcs \
--memory=512MB \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy csharp-finalize-function \
--gen2 \
--runtime=dotnet8 \
--region=REGION \
--source=. \
--entry-point=HelloGcs.Function \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy ruby-finalize-function \
--gen2 \
--runtime=ruby33 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy php-finalize-function \
--gen2 \
--runtime=php83 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

Replace the following:

• REGION: The name of the Google Cloud region where you want to deploy your function (for example, us-west1).
• YOUR_BUCKET_NAME: The name of the Cloud Storage bucket that triggers the function. When deploying Cloud Run functions, specify the bucket name alone without the leading gs://; for example, --trigger-event-filters="bucket=my-bucket".

Object Finalize: trigger the function

Test the function by uploading a file to your bucket:

echo "Hello World" > test-finalize.txt
gcloud storage cp test-finalize.txt gs://YOUR_BUCKET_NAME/test-finalize.txt

You should see the received CloudEvent in the logs:

gcloud functions logs read YOUR_FUNCTION_NAME --region REGION --gen2 --limit=10

Object Delete

Object delete events are most useful for non-versioning buckets. They are triggered when an old version of an object is deleted. In addition, they are triggered when an object is overwritten. Object delete triggers can also be used with versioning buckets, triggering when a version of an object is permanently deleted.

Object Delete: deploy the function

Using the same sample code as in the finalize example, deploy the function with object delete as the trigger event. Run the following command in the directory where the sample code is located:

gcloud functions deploy nodejs-delete-function \
--gen2 \
--runtime=nodejs22 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy python-delete-function \
--gen2 \
--runtime=python312 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy go-delete-function \
--gen2 \
--runtime=go122 \
--region=REGION \
--source=. \
--entry-point=HelloStorage \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy java-delete-function \
--gen2 \
--runtime=java21 \
--region=REGION \
--source=. \
--entry-point=functions.HelloGcs \
--memory=512MB \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy csharp-delete-function \
--gen2 \
--runtime=dotnet8 \
--region=REGION \
--source=. \
--entry-point=HelloGcs.Function \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy ruby-delete-function \
--gen2 \
--runtime=ruby33 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy php-delete-function \
--gen2 \
--runtime=php83 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.deleted" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

Replace the following:

• REGION: The name of the Google Cloud region where you want to deploy your function (for example, us-west1).
• YOUR_BUCKET_NAME: The name of the Cloud Storage bucket that triggers the function. When deploying Cloud Run functions, specify the bucket name alone without the leading gs://; for example, --trigger-event-filters="bucket=my-bucket".

Object Delete: trigger the function

To trigger the function:

1. Create an empty test-delete.txt file in the directory where the sample code is located.

2. Make sure that your bucket is non-versioning:

   gcloud storage buckets update gs://YOUR_BUCKET_NAME --no-versioning

3. Upload the file to Cloud Storage:

   gcloud storage cp test-delete.txt gs://YOUR_BUCKET_NAME

   where YOUR_BUCKET_NAME is the name of your Cloud Storage bucket where you will upload a test file. At this point the function should not execute yet.

4. Delete the file to trigger the function:

   gcloud storage rm gs://YOUR_BUCKET_NAME/test-delete.txt

5. You should see the received CloudEvent in the logs:

   gcloud functions logs read YOUR_FUNCTION_NAME --region REGION --gen2 --limit=10

Note that the function may take some time to finish executing.

Object Archive

Object archive events can be used only with versioning buckets. They are triggered when an earlier version of an object is archived. In particular, this means that when an object is overwritten or deleted, an archive event is triggered.

Object Archive: deploy the function

Using the same sample code as in the finalize example, deploy the function with object archive as the trigger event. Run the following command in the directory where the sample code is located:

gcloud functions deploy nodejs-archive-function \
--gen2 \
--runtime=nodejs22 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy python-archive-function \
--gen2 \
--runtime=python312 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy go-archive-function \
--gen2 \
--runtime=go122 \
--region=REGION \
--source=. \
--entry-point=HelloStorage \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy java-archive-function \
--gen2 \
--runtime=java21 \
--region=REGION \
--source=. \
--entry-point=functions.HelloGcs \
--memory=512MB \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy csharp-archive-function \
--gen2 \
--runtime=dotnet8 \
--region=REGION \
--source=. \
--entry-point=HelloGcs.Function \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy ruby-archive-function \
--gen2 \
--runtime=ruby33 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy php-archive-function \
--gen2 \
--runtime=php83 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.archived" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

Replace the following:

• REGION: The name of the Google Cloud region where you want to deploy your function (for example, us-west1).
• YOUR_BUCKET_NAME: The name of the Cloud Storage bucket that triggers the function. When deploying Cloud Run functions, specify the bucket name alone without the leading gs://; for example, --trigger-event-filters="bucket=my-bucket".

Object Archive: trigger the function

To trigger the function:

1. Create an empty test-archive.txt file in the directory where the sample code is located.

2. Make sure that your bucket has versioning enabled:

    gcloud storage buckets update gs://YOUR_BUCKET_NAME --versioning

3. Upload the file to Cloud Storage:

   gcloud storage cp test-archive.txt gs://YOUR_BUCKET_NAME

   where YOUR_BUCKET_NAME is the name of your Cloud Storage bucket where you will upload a test file. At this point the function should not execute yet.

4. Archive the file to trigger the function:

   gcloud storage rm gs://YOUR_BUCKET_NAME/test-archive.txt

5. You should see the received CloudEvent in the logs:

   gcloud functions logs read YOUR_FUNCTION_NAME --region REGION --gen2 --limit=10

Object Metadata Update

Metadata update events are triggered when the metadata of existing object is updated.

Object Metadata Update: deploy the function

Using the same sample code as in the finalize example, deploy the function with metadata update as the trigger event. Run the following command in the directory where the sample code is located:

gcloud functions deploy nodejs-metadata-function \
--gen2 \
--runtime=nodejs22 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy python-metadata-function \
--gen2 \
--runtime=python312 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy go-metadata-function \
--gen2 \
--runtime=go122 \
--region=REGION \
--source=. \
--entry-point=HelloStorage \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy java-metadata-function \
--gen2 \
--runtime=java21 \
--region=REGION \
--source=. \
--entry-point=functions.HelloGcs \
--memory=512MB \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy csharp-metadata-function \
--gen2 \
--runtime=dotnet8 \
--region=REGION \
--source=. \
--entry-point=HelloGcs.Function \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy ruby-metadata-function \
--gen2 \
--runtime=ruby33 \
--region=REGION \
--source=. \
--entry-point=hello_gcs \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

gcloud functions deploy php-metadata-function \
--gen2 \
--runtime=php83 \
--region=REGION \
--source=. \
--entry-point=helloGCS \
--trigger-event-filters="type=google.cloud.storage.object.v1.metadataUpdated" \
--trigger-event-filters="bucket=YOUR_BUCKET_NAME"

Replace the following:

• REGION: The name of the Google Cloud region where you want to deploy your function (for example, us-west1).
• YOUR_BUCKET_NAME: The name of the Cloud Storage bucket that triggers the function. When deploying Cloud Run functions, specify the bucket name alone without the leading gs://; for example, --trigger-event-filters="bucket=my-bucket".

Object Metadata Update: trigger the function

To trigger the function:

1. Create an empty test-metadata.txt file in the directory where the sample code is located.

2. Make sure that your bucket is non-versioning:

   gcloud storage buckets update gs://YOUR_BUCKET_NAME --no-versioning

3. Upload the file to Cloud Storage:

   gcloud storage cp test-metadata.txt gs://YOUR_BUCKET_NAME

   where YOUR_BUCKET_NAME is the name of your Cloud Storage bucket where you will upload a test file. At this point the function should not execute yet.

4. Update the metadata of the file:

   gcloud storage objects update gs://YOUR_BUCKET_NAME/test-metadata.txt --content-type=text/plain

5. You should see the received CloudEvent in the logs:

   gcloud functions logs read YOUR_FUNCTION_NAME --region REGION --gen2 --limit=10