Handle AWS Cognito Logs

pkg/cognito.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/grafana/loki/v3/pkg/logproto"
+)
+
+// Parses a Cognito Record and returns a logproto.Entry
+func parseCognitoRecord(record Record) (logproto.Entry, error) {
+	timestamp := time.Now()
+	if record.Error != nil {
+		return logproto.Entry{}, record.Error
+	}
+	document, err := json.Marshal(record.Content)
+	if err != nil {
+		return logproto.Entry{}, err
+	}
+	if val, ok := record.Content["eventTimestamp"]; ok {
+		sec, nsec, err := getUnixSecNsec(val.(string))
+		if err != nil {
+			return logproto.Entry{}, err
+		}
+
+		timestamp = time.Unix(sec, nsec).UTC()
+	}
+	return logproto.Entry{
+		Line:      string(document),
+		Timestamp: timestamp,
+	}, nil
+}

pkg/cognito_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"compress/gzip"
+	"os"
+	"testing"
+)
+
+func TestParseCognitoJson(t *testing.T) {
+	records := make(chan Record)
+	jsonStream := NewJSONStream(records)
+	file, err := os.Open("../testdata/cognito-log-file.json.gz")
+	if err != nil {
+		t.Error(err)
+	}
+	gzipReader, err := gzip.NewReader(file)
+	if err != nil {
+		t.Error(err)
+	}
+	go jsonStream.Start(gzipReader, 0)
+
+	for record := range jsonStream.records {
+		if record.Error != nil {
+			t.Error(record.Error)
+		}
+		_, err := parseCognitoRecord(record)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+}

pkg/s3.go

@@ -55,6 +55,7 @@ const (
 	GuardDutyLogType        string = "GuardDuty"
 	MskLogType              string = "KafkaBrokerLogs"
 	S3AccessLogType         string = "S3AccessLogs"
+	CognitoServiceLogsType  string = "AWSCognitoServiceLogs"
 )
 
 var (
@@ -92,6 +93,10 @@ var (
 	// source: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html
 	// format: aws-account-id/region/bucket-name/year/month/day/timestamp-hash
 	// example: 123456789012/us-west-2/amzn-s3-demo-source-bucket/2023/03/01/2023-03-01-21-32-16-E568B2907131C0C0
+	// AWS Cognito Service Logs
+	// source: https://docs.aws.amazon.com/cognito/latest/developerguide/monitoring.html
+	// format: my-bucket/AWSLogs/aws-account-id/AWSCognitoServiceLogs/region/cognito-pool-id/event-type/random-string.log.gz
+	// example: my-bucket/AWSLogs/116911669293/AWSCognitoServiceLogs/eu-central-1/eu-central-1-xxx/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-xxx_userAuthEvents_INFO_S3_yyy_cognito-eu-central-1-xxx_USER_AUTH_EVENTS_20251216_ccc.log.gz
 	defaultFilenameRegex      = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?P<lb_type>app|net)\.*?)?(?P<src>[a-zA-Z0-9\-]+)`)
 	defaultTimestampRegex     = regexp.MustCompile(`(?P<timestamp>\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`)
 	cloudtrailFilenameRegex   = regexp.MustCompile(`AWSLogs\/(?P<organization_id>o-[a-z0-9]{10,32})?\/?(?P<account_id>\d+)\/(?P<type>[a-zA-Z0-9_\-]+)\/(?P<region>[\w-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P<src>[a-zA-Z0-9\-]+)`)
@@ -104,6 +109,7 @@ var (
 	mskTimestampRegex         = regexp.MustCompile(`^\[(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]`)
 	s3AccessLogFilenameRegex  = regexp.MustCompile(`(?P<account_id>\d+)\/(?P<region>[\w-]+)\/(?P<src>[a-zA-Z0-9\-]+)\/(?P<year>\d+)\/(?P<month>\d+)\/(?P<day>\d+)\/[a-zA-Z0-9\-]+$`)
 	s3AccessLogTimestampRegex = regexp.MustCompile(`\[(?P<timestamp>\d+\/\w+\/\d+:\d+:\d+:\d+ [\+-]\d+)\]`)
+	cognitoServiceLogsRegex   = regexp.MustCompile(`AWSLogs\/(?P<account_id>\d+)\/AWSCognitoServiceLogs\/(?P<region>[\w-]+)\/(?P<cognito_pool_id>[a-zA-Z0-9_\-]+)\/(?P<event_type_id>[a-zA-Z0-9_\-]+)\/(?P<src>\d+_aws_cognito_service_logs_[a-zA-Z0-9_\-]+\.log\.gz)`)
 	parsers                   = map[string]parserConfig{
 		FlowLogType: {
 			logTypeLabel:    "s3_vpc_flow",
@@ -169,6 +175,11 @@ var (
 			timestampRegex:  s3AccessLogTimestampRegex,
 			timestampType:   "string",
 		},
+		CognitoServiceLogsType: {
+			logTypeLabel:  "s3_cognito_service_logs",
+			filenameRegex: cognitoServiceLogsRegex,
+			ownerLabelKey: "account_id",
+		},
 	}
 )
 
@@ -221,7 +232,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
 	ls = applyLabels(ls)
 
 	// extract the timestamp of the nested event and sends the rest as raw json
-	if labels["type"] == CloudTrailLogType || labels["type"] == GuardDutyLogType {
+	if labels["type"] == CloudTrailLogType || labels["type"] == GuardDutyLogType || labels["type"] == CognitoServiceLogsType {
 		records := make(chan Record)
 		jsonStream := NewJSONStream(records)
 		go jsonStream.Start(reader, parser.skipHeaderCount)
@@ -230,11 +241,18 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io.
 			if record.Error != nil {
 				return record.Error
 			}
-			trailEntry, err := parseCloudtrailRecord(record)
+
+			var objEntry logproto.Entry
+			if labels["type"] == CognitoServiceLogsType {
+				objEntry, err = parseCognitoRecord(record)
+			} else {
+				objEntry, err = parseCloudtrailRecord(record)
+			}
+
 			if err != nil {
 				return err
 			}
-			if err := b.add(ctx, entry{ls, trailEntry}); err != nil {
+			if err := b.add(ctx, entry{ls, objEntry}); err != nil {
 				return err
 			}
 		}

pkg/s3_test.go

@@ -590,6 +590,38 @@ func Test_getLabels(t *testing.T) {
 			},
 			wantErr: true,
 		},
+		{
+			name: "cognito_service_logs",
+			args: args{
+				record: events.S3EventRecord{
+					AWSRegion: "eu-central-1",
+					S3: events.S3Entity{
+						Bucket: events.S3Bucket{
+							Name: "my-bucket",
+							OwnerIdentity: events.S3UserIdentity{
+								PrincipalID: "test",
+							},
+						},
+						Object: events.S3Object{
+							Key: "my-bucket/AWSLogs/123456789012/AWSCognitoServiceLogs/eu-central-1/eu-central-1-manual-testing/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz",
+						},
+					},
+				},
+			},
+			want: map[string]string{
+				"account_id":      "123456789012",
+				"bucket":          "my-bucket",
+				"bucket_owner":    "test",
+				"bucket_region":   "eu-central-1",
+				"cognito_pool_id": "eu-central-1-manual-testing",
+				"event_type_id":   "USER_AUTH_EVENTS",
+				"key":             "my-bucket/AWSLogs/123456789012/AWSCognitoServiceLogs/eu-central-1/eu-central-1-manual-testing/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz",
+				"region":          "eu-central-1",
+				"src":             "123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz",
+				"type":            CognitoServiceLogsType,
+			},
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -866,6 +898,29 @@ func Test_parseS3Log(t *testing.T) {
 			expectedLog:    `level=warn msg="timestamp type of no_type parser unknown, using current time"` + "\n",
 			wantErr:        false,
 		},
+		{
+			name: "cognitoservicelogs",
+			args: args{
+				batchSize: 131072, // Set large enough we don't try and send to promtail
+				filename:  "../testdata/cognito-log-file.json.gz",
+				b: &batch{
+					streams:   map[string]*logproto.Stream{},
+					processor: process,
+				},
+				labels: map[string]string{
+					"type":       CognitoServiceLogsType,
+					"src":        "source",
+					"account_id": "123456789",
+				},
+			},
+			expectedLen:    1,
+			expectedStream: `{__aws_log_type="s3_cognito_service_logs", __aws_s3_cognito_service_logs="source", __aws_s3_cognito_service_logs_owner="123456789"}`,
+			expectedTimestamps: []time.Time{
+				time.Date(2025, time.December, 16, 8, 47, 05, 0, time.UTC),
+				time.Date(2025, time.December, 16, 8, 47, 10, 0, time.UTC),
+			},
+			wantErr: false,
+		},
 	}
 	parsers["no_type"] = parserConfig{
 		logTypeLabel:   "s3_waf",