[WIP] Make add-metric-suffixes configurable

CHANGELOG.md

@@ -9,6 +9,7 @@
   * Metrics: Renamed `cortex_parquet_queryable_cache_*` to `cortex_parquet_cache_*`.
   * Flags: Renamed `-querier.parquet-queryable-shard-cache-size` to `-querier.parquet-shard-cache-size` and `-querier.parquet-queryable-shard-cache-ttl` to `-querier.parquet-shard-cache-ttl`.
   * Config: Renamed `parquet_queryable_shard_cache_size` to `parquet_shard_cache_size` and `parquet_queryable_shard_cache_ttl` to `parquet_shard_cache_ttl`.
+* [FEATURE] Distrubutor: Add `-distributor.otlp.add-metric-suffixes` flag. If true, suffixes will be added to the metrics. #7286
 * [FEATURE] StoreGateway: Introduces a new parquet mode. #7046
 * [FEATURE] StoreGateway: Add a parquet shard cache to parquet mode. #7166
 * [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` that enables adding `__unit__` and `__type__` labels for remote write v2 and OTLP requests. This is a breaking change; the `-distributor.otlp.enable-type-and-unit-labels` flag is now deprecated, operates as a no-op, and has been consolidated into this new flag. #7077

docs/configuration/config-file-reference.md

@@ -3337,6 +3337,10 @@ otlp:
   # Deprecated: Use `-distributor.enable-type-and-unit-labels` flag instead.
   # CLI flag: -distributor.otlp.enable-type-and-unit-labels
   [enable_type_and_unit_labels: <boolean> | default = false]
+
+  # If true, suffixes will be added to the metrics.
+  # CLI flag: -distributor.otlp.add-metric-suffixes
+  [add_metric_suffixes: <boolean> | default = true]
 ```
 
 ### `etcd_config`

docs/guides/open-telemetry-collector.md

@@ -77,6 +77,7 @@ distributor:
     disable_target_info: <boolean>
     allow_delta_temporality: <boolean>
     enable_type_and_unit_labels: <boolean>
+    add_metric_suffixes: <boolean> // pending more here
 ```
 
 ### Ingest `target_info` metric
@@ -156,3 +157,8 @@ overrides:
     promote_resource_attributes: ["attr1", "attr2"]
 `
 ```
+
+### Configure adding suffixes to metrics
+
+The flag `add_metric_suffixes` allows control to add suffixes to metrics for normalization.
+This flag is enabled by default.

pkg/distributor/distributor.go

@@ -202,6 +202,7 @@ type OTLPConfig struct {
 	DisableTargetInfo       bool `yaml:"disable_target_info"`
 	AllowDeltaTemporality   bool `yaml:"allow_delta_temporality"`
 	EnableTypeAndUnitLabels bool `yaml:"enable_type_and_unit_labels"`
+	AddMetricSuffixes       bool `yaml:"add_metric_suffixes"`
 }
 
 // RegisterFlags adds the flags required to config this to the given FlagSet
@@ -231,6 +232,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.OTLPConfig.DisableTargetInfo, "distributor.otlp.disable-target-info", false, "If true, a target_info metric is not ingested. (refer to: https://github.com/prometheus/OpenMetrics/blob/main/specification/OpenMetrics.md#supporting-target-metadata-in-both-push-based-and-pull-based-systems)")
 	f.BoolVar(&cfg.OTLPConfig.AllowDeltaTemporality, "distributor.otlp.allow-delta-temporality", false, "EXPERIMENTAL: If true, delta temporality otlp metrics to be ingested.")
 	f.BoolVar(&cfg.OTLPConfig.EnableTypeAndUnitLabels, "distributor.otlp.enable-type-and-unit-labels", false, "Deprecated: Use `-distributor.enable-type-and-unit-labels` flag instead.")
+	f.BoolVar(&cfg.OTLPConfig.AddMetricSuffixes, "distributor.otlp.add-metric-suffixes", true, "If true, suffixes will be added to all metrics.")
 }
 
 // Validate config and returns error on failure

pkg/util/push/otlp.go

@@ -184,7 +184,7 @@ func convertToPromTS(ctx context.Context, pmetrics pmetric.Metrics, cfg distribu
 	collector := newCollectingAppender()
 	promConverter := prometheusremotewrite.NewPrometheusConverter(collector)
 	settings := prometheusremotewrite.Settings{
-		AddMetricSuffixes:       true,
+		AddMetricSuffixes:       cfg.AddMetricSuffixes,
 		DisableTargetInfo:       cfg.DisableTargetInfo,
 		AllowDeltaTemporality:   cfg.AllowDeltaTemporality,
 		EnableTypeAndUnitLabels: overrides.EnableTypeAndUnitLabels(userID),

pkg/util/push/otlp_test.go

@@ -39,13 +39,15 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		description             string
 		enableTypeAndUnitLabels bool
 		allowDeltaTemporality   bool
+		addMetricSuffixes       bool
 		otlpSeries              pmetric.Metric
 		expectedLabels          labels.Labels
 		expectedMetadata        prompb.MetricMetadata
 	}{
 		{
 			description:             "[enableTypeAndUnitLabels: true], the '__type__' label should be attached when the type is the gauge",
 			enableTypeAndUnitLabels: true,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -59,6 +61,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 			description:             "[enableTypeAndUnitLabels: true], the '__type__' label should not be attached when the type is unknown",
 			enableTypeAndUnitLabels: true,
 			allowDeltaTemporality:   true,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityDelta, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -70,6 +73,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		{
 			description:             "[enableTypeAndUnitLabels: false]",
 			enableTypeAndUnitLabels: false,
+			addMetricSuffixes:       true,
 			otlpSeries:              createOtelSum("test", "seconds", pmetric.AggregationTemporalityCumulative, ts),
 			expectedLabels: labels.FromMap(map[string]string{
 				"__name__":   "test_seconds",
@@ -83,6 +87,7 @@ func TestOTLP_EnableTypeAndUnitLabels(t *testing.T) {
 		t.Run(test.description, func(t *testing.T) {
 			cfg := distributor.OTLPConfig{
 				AllowDeltaTemporality: test.allowDeltaTemporality,
+				AddMetricSuffixes:     test.addMetricSuffixes,
 			}
 			metrics := pmetric.NewMetrics()
 			rm := metrics.ResourceMetrics().AppendEmpty()
@@ -388,6 +393,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    false,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -410,6 +416,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -432,6 +439,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: false,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -450,6 +458,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: true,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{
@@ -484,6 +493,7 @@ func TestOTLPConvertToPromTS(t *testing.T) {
 			cfg: distributor.OTLPConfig{
 				ConvertAllAttributes: true,
 				DisableTargetInfo:    true,
+				AddMetricSuffixes:    true,
 			},
 			expectedLabels: []prompb.Label{
 				{

schemas/cortex-config-schema.json

@@ -3906,6 +3906,12 @@
               "description": "Deprecated: Use `-distributor.enable-type-and-unit-labels` flag instead.",
               "type": "boolean",
               "x-cli-flag": "distributor.otlp.enable-type-and-unit-labels"
+            },
+            "add_metric_suffixes": {
+              "default": true,
+              "description": "If true, suffixes will be added to the metrics.",
+              "type": "boolean",
+              "x-cli-flag": "distributor.otlp.add-metric-suffixes"
             }
           },
           "type": "object"