diff --git a/config/headers.go b/config/headers.go
index 9beaae26..a66d5f9f 100644
--- a/config/headers.go
+++ b/config/headers.go
@@ -64,8 +64,9 @@ func (h *Headers) SetDirectory(dir string) {
 	if h == nil {
 		return
 	}
-	for _, h := range h.Headers {
-		h.SetDirectory(dir)
+	for name, header := range h.Headers {
+		header.SetDirectory(dir)
+		h.Headers[name] = header
 	}
 }
 
diff --git a/config/headers_test.go b/config/headers_test.go
index daac9808..b156427d 100644
--- a/config/headers_test.go
+++ b/config/headers_test.go
@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -37,6 +38,23 @@ func TestReservedHeaders(t *testing.T) {
 	}
 }
 
+func TestHeadersSetDirectory(t *testing.T) {
+	headers := &Headers{
+		Headers: map[string]Header{
+			"X-Test": {
+				Files: []string{"headers-file-a", "/tmp/already-absolute"},
+			},
+		},
+	}
+
+	headers.SetDirectory("/etc/prometheus")
+
+	require.Equal(t,
+		[]string{filepath.Join("/etc/prometheus", "headers-file-a"), "/tmp/already-absolute"},
+		headers.Headers["X-Test"].Files,
+	)
+}
+
 func TestHeadersRoundTripperSameHost(t *testing.T) {
 	// All headers, including sensitive ones, must be forwarded on same-host requests.
 	for _, header := range []string{"Cookie", "X-Custom-Header"} {
diff --git a/config/http_config_test.go b/config/http_config_test.go
index 708e993b..0a9debd0 100644
--- a/config/http_config_test.go
+++ b/config/http_config_test.go
@@ -2720,3 +2720,28 @@ func TestMultipleHeaders(t *testing.T) {
 	_, err = client.Get(ts.URL)
 	require.NoErrorf(t, err, "can't fetch URL: %v", err)
 }
+
+func TestHeadersFileRelativeToConfigFile(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "nested-value", r.Header.Get("X-Test"))
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	t.Cleanup(ts.Close)
+
+	tmpDir := t.TempDir()
+	configDir := filepath.Join(tmpDir, "configs")
+	require.NoError(t, os.MkdirAll(configDir, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "header-value"), []byte("nested-value\n"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(configDir, "http.yml"), []byte(`http_headers:
+  X-Test:
+    files: [configs/header-value]
+`), 0o644))
+
+	cfg, _, err := LoadHTTPConfigFile(filepath.Join(configDir, "http.yml"))
+	require.NoErrorf(t, err, "Error loading HTTP client config: %v", err)
+	client, err := NewClientFromConfig(*cfg, "test")
+	require.NoErrorf(t, err, "Error creating HTTP Client: %v", err)
+
+	_, err = client.Get(ts.URL)
+	require.NoErrorf(t, err, "can't fetch URL: %v", err)
+}