diff --git a/.github/actions/ci/action.yml b/.github/actions/ci/action.yml index 35c1ca23b..86f8f7893 100644 --- a/.github/actions/ci/action.yml +++ b/.github/actions/ci/action.yml @@ -37,6 +37,14 @@ inputs: description: 'Whether to install CURL development libraries. Required for OpenTelemetry builds (server-sdk-otel), but does not enable CURL networking for the SDK itself.' required: false default: 'false' + use_redis: + description: 'Whether to enable Redis support (LD_BUILD_REDIS_SUPPORT=ON). Leave unset to preserve automatic detection for Redis targets.' + required: false + default: '' + use_dynamodb: + description: 'Whether to enable DynamoDB support (LD_BUILD_DYNAMODB_SUPPORT=ON). Leave unset to preserve automatic detection for DynamoDB targets.' + required: false + default: '' runs: using: composite @@ -58,7 +66,7 @@ runs: id: install-curl - name: Build Library shell: bash - run: ./scripts/build.sh ${{ inputs.cmake_target }} ON ${{ inputs.use_curl }} + run: ./scripts/build.sh ${{ inputs.cmake_target }} ON "${{ inputs.use_curl }}" "${{ inputs.use_redis }}" "${{ inputs.use_dynamodb }}" env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} Boost_DIR: ${{ steps.install-boost.outputs.Boost_DIR }} @@ -69,7 +77,7 @@ runs: id: build-tests if: inputs.run_tests == 'true' shell: bash - run: ./scripts/build.sh gtest_${{ inputs.cmake_target }} ON ${{ inputs.use_curl }} + run: ./scripts/build.sh gtest_${{ inputs.cmake_target }} ON "${{ inputs.use_curl }}" "${{ inputs.use_redis }}" "${{ inputs.use_dynamodb }}" env: BOOST_ROOT: ${{ steps.install-boost.outputs.BOOST_ROOT }} Boost_DIR: ${{ steps.install-boost.outputs.Boost_DIR }} diff --git a/.github/workflows/server-dynamodb.yml b/.github/workflows/server-dynamodb.yml index 94e1a26c6..3ff1b6fba 100644 --- a/.github/workflows/server-dynamodb.yml +++ b/.github/workflows/server-dynamodb.yml @@ -14,6 +14,36 @@ on: - cron: '0 8 * * *' jobs: + contract-tests-dynamodb: + runs-on: ubuntu-22.04 + services: + dynamodb: + image: amazon/dynamodb-local + ports: + - 8000:8000 + env: + TEST_SERVICE_PORT: 8123 + TEST_SERVICE_BINARY: ./build/contract-tests/server-contract-tests/server-tests + steps: + # https://github.com/actions/checkout/releases/tag/v4.3.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + - uses: ./.github/actions/ci + with: + cmake_target: server-tests + run_tests: false + # AWS C++ SDK requires libcurl at link time on Linux/macOS. + install_curl: true + use_dynamodb: true + - name: 'Launch test service as background task' + run: $TEST_SERVICE_BINARY $TEST_SERVICE_PORT 2>&1 & + # https://github.com/launchdarkly/gh-actions/releases/tag/contract-tests-v1.3.0 + - uses: launchdarkly/gh-actions/actions/contract-tests@5adb11fd6953e1bc35d9cf1fc1b4374c464e3a8b + with: + test_service_port: ${{ env.TEST_SERVICE_PORT }} + token: ${{ secrets.GITHUB_TOKEN }} + enable_persistence_tests: true + extra_params: '--skip-from ./contract-tests/server-contract-tests/suppressions.txt' + build-test-dynamodb: runs-on: ubuntu-22.04 services: diff --git a/.github/workflows/server-redis.yml b/.github/workflows/server-redis.yml index 8b4d90ccc..e32b8014f 100644 --- a/.github/workflows/server-redis.yml +++ b/.github/workflows/server-redis.yml @@ -14,6 +14,63 @@ on: - cron: '0 8 * * *' jobs: + contract-tests: + runs-on: ubuntu-22.04 + services: + redis: + image: redis + ports: + - 6379:6379 + env: + TEST_SERVICE_PORT: 8123 + TEST_SERVICE_BINARY: ./build/contract-tests/server-contract-tests/server-tests + steps: + # https://github.com/actions/checkout/releases/tag/v4.3.0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 + - uses: ./.github/actions/ci + with: + cmake_target: server-tests + run_tests: false + use_redis: true + - name: 'Launch test service as background task' + run: $TEST_SERVICE_BINARY $TEST_SERVICE_PORT 2>&1 & + # https://github.com/launchdarkly/gh-actions/releases/tag/contract-tests-v1.3.0 + - uses: launchdarkly/gh-actions/actions/contract-tests@5adb11fd6953e1bc35d9cf1fc1b4374c464e3a8b + with: + test_service_port: ${{ env.TEST_SERVICE_PORT }} + token: ${{ secrets.GITHUB_TOKEN }} + enable_persistence_tests: true + extra_params: '--skip-from ./contract-tests/server-contract-tests/suppressions.txt' + + contract-tests-curl: + runs-on: ubuntu-22.04 + services: + redis: + image: redis + ports: + - 6379:6379 + env: + TEST_SERVICE_PORT: 8123 + TEST_SERVICE_BINARY: ./build/contract-tests/server-contract-tests/server-tests + steps: + # https://github.com/actions/checkout/releases/tag/v4.3.0 + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 + - uses: ./.github/actions/ci + with: + cmake_target: server-tests + run_tests: false + use_curl: true + use_redis: true + - name: 'Launch test service as background task' + run: $TEST_SERVICE_BINARY $TEST_SERVICE_PORT 2>&1 & + # https://github.com/launchdarkly/gh-actions/releases/tag/contract-tests-v1.3.0 + - uses: launchdarkly/gh-actions/actions/contract-tests@5adb11fd6953e1bc35d9cf1fc1b4374c464e3a8b + with: + test_service_port: ${{ env.TEST_SERVICE_PORT }} + token: ${{ secrets.GITHUB_TOKEN }} + enable_persistence_tests: true + extra_params: '--skip-from ./contract-tests/server-contract-tests/suppressions.txt' + build-test-redis: runs-on: ubuntu-22.04 services: @@ -28,6 +85,7 @@ jobs: with: cmake_target: launchdarkly-cpp-server-redis-source simulate_release: true + use_redis: true build-redis-mac: runs-on: macos-15 steps: @@ -39,6 +97,7 @@ jobs: platform_version: 12 run_tests: false # TODO: figure out how to run Redis service on Mac simulate_release: true + use_redis: true build-test-redis-windows: runs-on: windows-2022 steps: @@ -57,3 +116,4 @@ jobs: toolset: msvc run_tests: false # TODO: figure out how to run Redis service on Windows simulate_windows_release: true + use_redis: true diff --git a/contract-tests/data-model/include/data_model/data_model.hpp b/contract-tests/data-model/include/data_model/data_model.hpp index 60ce88295..5ff64790f 100644 --- a/contract-tests/data-model/include/data_model/data_model.hpp +++ b/contract-tests/data-model/include/data_model/data_model.hpp @@ -149,6 +149,35 @@ struct ConfigWrapper { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConfigWrapper, name, version); +struct ConfigPersistentCache { + std::string mode; // "off", "ttl", "infinite" + std::optional ttl; // TTL in seconds (sent by test harness as "ttl") +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConfigPersistentCache, + mode, + ttl); + +struct ConfigPersistentStore { + std::string type; // "redis", "consul", "dynamodb" + std::string dsn; + std::optional prefix; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConfigPersistentStore, + type, + dsn, + prefix); + +struct ConfigPersistentDataStore { + ConfigPersistentStore store; + ConfigPersistentCache cache; +}; + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConfigPersistentDataStore, + store, + cache); + struct ConfigDataSynchronizerParams { std::optional streaming; std::optional polling; @@ -208,6 +237,7 @@ struct ConfigParams { std::optional proxy; std::optional hooks; std::optional wrapper; + std::optional persistentDataStore; std::optional bigSegments; }; @@ -226,6 +256,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(ConfigParams, proxy, hooks, wrapper, + persistentDataStore, bigSegments); struct ContextSingleParams { diff --git a/contract-tests/server-contract-tests/CMakeLists.txt b/contract-tests/server-contract-tests/CMakeLists.txt index 000dea286..5b78d4f03 100644 --- a/contract-tests/server-contract-tests/CMakeLists.txt +++ b/contract-tests/server-contract-tests/CMakeLists.txt @@ -29,4 +29,14 @@ target_link_libraries(server-tests PRIVATE contract-test-data-model ) +if (LD_BUILD_REDIS_SUPPORT) + target_link_libraries(server-tests PRIVATE launchdarkly::server_redis_source) + target_compile_definitions(server-tests PRIVATE LD_REDIS_SUPPORT_ENABLED) +endif () + +if (LD_BUILD_DYNAMODB_SUPPORT) + target_link_libraries(server-tests PRIVATE launchdarkly::server_dynamodb_source) + target_compile_definitions(server-tests PRIVATE LD_DYNAMODB_SUPPORT_ENABLED) +endif () + target_include_directories(server-tests PUBLIC include) diff --git a/contract-tests/server-contract-tests/src/entity_manager.cpp b/contract-tests/server-contract-tests/src/entity_manager.cpp index 6536c6b87..ffbdfdf11 100644 --- a/contract-tests/server-contract-tests/src/entity_manager.cpp +++ b/contract-tests/server-contract-tests/src/entity_manager.cpp @@ -9,11 +9,75 @@ #include +#ifdef LD_REDIS_SUPPORT_ENABLED +#include +#endif + +#ifdef LD_DYNAMODB_SUPPORT_ENABLED +#include +#endif + using launchdarkly::LogLevel; using namespace launchdarkly::server_side; +#if defined(LD_REDIS_SUPPORT_ENABLED) || defined(LD_DYNAMODB_SUPPORT_ENABLED) +#define LD_PERSISTENT_STORE_ENABLED +#endif + namespace { +#ifdef LD_PERSISTENT_STORE_ENABLED +tl::expected +MakePersistentStoreSource(ConfigPersistentStore const& store) { +#ifdef LD_REDIS_SUPPORT_ENABLED + if (store.type == "redis") { + auto source = integrations::RedisDataSource::Create( + store.dsn, store.prefix.value_or("launchdarkly")); + if (!source) { + return tl::make_unexpected(source.error()); + } + return std::move(*source); + } +#endif +#ifdef LD_DYNAMODB_SUPPORT_ENABLED + if (store.type == "dynamodb") { + // The test harness creates this table and provides its endpoint as the + // DSN. Credentials are required by the AWS SDK, but ignored by + // DynamoDB Local. + integrations::DynamoDBClientOptions options; + options.region = "us-east-1"; + options.endpoint = store.dsn; + options.aws_access_key_id = "dummy"; + options.aws_secret_access_key = "dummy"; + options.aws_session_token = "dummy"; + + auto source = integrations::DynamoDBDataSource::Create( + "sdk-contract-tests", store.prefix.value_or(""), + std::move(options)); + if (!source) { + return tl::make_unexpected(source.error()); + } + return std::move(*source); + } +#endif + return tl::make_unexpected("unsupported persistent store type: " + + store.type); +} + +void ApplyCacheConfig(config::builders::LazyLoadBuilder& lazy_load, + ConfigPersistentCache const& cache) { + if (cache.mode == "off") { + lazy_load.CacheRefresh(std::chrono::seconds(0)); + } else if (cache.mode == "ttl") { + if (cache.ttl) { + lazy_load.CacheRefresh(std::chrono::seconds(*cache.ttl)); + } + } else if (cache.mode == "infinite") { + lazy_load.CacheRefresh(std::chrono::hours(24 * 365)); + } +} +#endif + config::builders::DataSystemBuilder::FDv2 BuildFDv2( ConfigDataSystemParams const& cfg, config::builders::EndpointsBuilder* endpoints) { @@ -288,6 +352,26 @@ std::optional EntityManager::create(ConfigParams const& in) { } } +#ifdef LD_PERSISTENT_STORE_ENABLED + if (in.persistentDataStore) { + auto source = MakePersistentStoreSource(in.persistentDataStore->store); + if (!source) { + LD_LOG(logger_, LogLevel::kWarn) + << "entity_manager: couldn't create persistent store source: " + << source.error(); + return std::nullopt; + } + + auto lazy_load = config::builders::LazyLoadBuilder(); + lazy_load.Source(std::move(*source)); + ApplyCacheConfig(lazy_load, in.persistentDataStore->cache); + + config_builder.DataSystem().Method( + config::builders::DataSystemBuilder::LazyLoad( + std::move(lazy_load))); + } +#endif + if (in.bigSegments) { auto store = std::make_shared( in.bigSegments->callbackUri); diff --git a/contract-tests/server-contract-tests/src/main.cpp b/contract-tests/server-contract-tests/src/main.cpp index 2422623a3..6551699c6 100644 --- a/contract-tests/server-contract-tests/src/main.cpp +++ b/contract-tests/server-contract-tests/src/main.cpp @@ -54,6 +54,12 @@ int main(int argc, char* argv[]) { srv.add_capability("instance-id"); srv.add_capability("fdv1-fallback"); srv.add_capability("big-segments"); +#ifdef LD_REDIS_SUPPORT_ENABLED + srv.add_capability("persistent-data-store-redis"); +#endif +#ifdef LD_DYNAMODB_SUPPORT_ENABLED + srv.add_capability("persistent-data-store-dynamodb"); +#endif net::signal_set signals{ioc, SIGINT, SIGTERM}; diff --git a/contract-tests/server-contract-tests/suppressions.txt b/contract-tests/server-contract-tests/suppressions.txt new file mode 100644 index 000000000..707af9a2b --- /dev/null +++ b/contract-tests/server-contract-tests/suppressions.txt @@ -0,0 +1,24 @@ +persistent data store/redis/read-write/initializes store when data received +persistent data store/redis/read-write/applies updates to store +persistent data store/redis/read-write/data source updates respect versioning +persistent data store/redis/read-write/data source deletions respect versioning +persistent data store/redis/read-write/cache mode infinite/does not cache flag miss +persistent data store/redis/read-write/cache mode infinite/sdk reflects data source updates even with cache +persistent data store/redis/read-write/cache mode infinite/ignores direct database modifications +persistent data store/redis/read-write/cache mode infinite/ignores dropped flags +persistent data store/redis/read-write/cache mode ttl/does not cache flag miss +persistent data store/redis/read-write/cache mode ttl/sdk reflects data source updates even with cache +persistent data store/redis/read-write/cache mode ttl/ignores direct database modifications +persistent data store/redis/read-write/cache mode ttl/ignores dropped flags +persistent data store/dynamodb/read-write/initializes store when data received +persistent data store/dynamodb/read-write/applies updates to store +persistent data store/dynamodb/read-write/data source updates respect versioning +persistent data store/dynamodb/read-write/data source deletions respect versioning +persistent data store/dynamodb/read-write/cache mode infinite/does not cache flag miss +persistent data store/dynamodb/read-write/cache mode infinite/sdk reflects data source updates even with cache +persistent data store/dynamodb/read-write/cache mode infinite/ignores direct database modifications +persistent data store/dynamodb/read-write/cache mode infinite/ignores dropped flags +persistent data store/dynamodb/read-write/cache mode ttl/does not cache flag miss +persistent data store/dynamodb/read-write/cache mode ttl/sdk reflects data source updates even with cache +persistent data store/dynamodb/read-write/cache mode ttl/ignores direct database modifications +persistent data store/dynamodb/read-write/cache mode ttl/ignores dropped flags diff --git a/scripts/build.sh b/scripts/build.sh index 3d3dd1565..396475efd 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -7,6 +7,8 @@ # $1 the name of the target. For example "launchdarkly-cpp-common". # $2 ON/OFF which enables/disables building in a test configuration (unit tests + contract tests.) # $3 (optional) true/false to enable/disable CURL networking (LD_CURL_NETWORKING) +# $4 (optional) true/false to enable/disable Redis support (LD_BUILD_REDIS_SUPPORT) +# $5 (optional) true/false to enable/disable DynamoDB support (LD_BUILD_DYNAMODB_SUPPORT) function cleanup { cd .. @@ -37,6 +39,20 @@ if [ "$3" == "true" ]; then build_curl="ON" fi +# Check for Redis support option (override the automatic detection if explicitly passed) +if [ "$4" == "true" ]; then + build_redis="ON" +elif [ "$4" == "false" ]; then + build_redis="OFF" +fi + +# Check for DynamoDB support option (override the automatic detection if explicitly passed) +if [ "$5" == "true" ]; then + build_dynamodb="ON" +elif [ "$5" == "false" ]; then + build_dynamodb="OFF" +fi + # Set build type to Debug when testing is enabled build_type="Release" if [ "$2" == "ON" ]; then