diff --git a/docs/content/en/docs/documentation/operations/configuration.md b/docs/content/en/docs/documentation/operations/configuration.md index cae4ef686e..513cc432d8 100644 --- a/docs/content/en/docs/documentation/operations/configuration.md +++ b/docs/content/en/docs/documentation/operations/configuration.md @@ -324,6 +324,7 @@ All controller-level keys are prefixed with `josdk.controller.. | `josdk.controller..finalizer` | `String` | Finalizer string added to managed resources | | `josdk.controller..generation-aware` | `Boolean` | Skip reconciliation when the resource generation has not changed | | `josdk.controller..label-selector` | `String` | Label selector to filter watched resources | +| `josdk.controller..shard-selector` | `String` | Shard selector to filter watched resources for sharding across operator replicas | | `josdk.controller..max-reconciliation-interval` | `Duration` | Maximum interval between reconciliations even without events | | `josdk.controller..field-manager` | `String` | Field manager name used for SSA operations | | `josdk.controller..trigger-reconciler-on-all-events` | `Boolean` | Trigger reconciliation on every event, not only meaningful changes | @@ -333,6 +334,7 @@ All controller-level keys are prefixed with `josdk.controller.. | Key | Type | Description | |---|---|---| | `josdk.controller..informer.label-selector` | `String` | Label selector for the primary resource informer (alias for `label-selector`) | +| `josdk.controller..informer.shard-selector` | `String` | Shard selector for the primary resource informer (alias for `shard-selector`) | | `josdk.controller..informer.list-limit` | `Long` | Page size for paginated informer list requests; omit for no pagination | #### Retry diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverrider.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverrider.java index 555c2ec3d4..1c1e03c870 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverrider.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverrider.java @@ -136,6 +136,11 @@ public ControllerConfigurationOverrider withLabelSelector(String labelSelecto return this; } + public ControllerConfigurationOverrider withShardSelector(String shardSelector) { + config.withShardSelector(shardSelector); + return this; + } + public ControllerConfigurationOverrider withReconciliationMaxInterval( Duration reconciliationMaxInterval) { this.reconciliationMaxInterval = reconciliationMaxInterval; diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/Informer.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/Informer.java index 7f0d266684..04f97902d3 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/Informer.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/Informer.java @@ -71,6 +71,18 @@ */ String labelSelector() default NO_VALUE_SET; + /** + * Optional shard selector used to restrict the set of resources the associated informer will act + * upon to a single shard, typically when the same workload is split across several operator + * instances. Just like {@link #labelSelector()} it is expressed as a label selector and can be + * made of multiple comma separated requirements that act as a logical AND operator. When both a + * label selector and a shard selector are set, the resulting informer only watches resources + * matching both (the two selectors are combined with a logical AND). + * + * @return the shard selector + */ + String shardSelector() default NO_VALUE_SET; + /** * Optional {@link OnAddFilter} to filter add events sent to the associated informer * diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java index 20d7df7136..6c92dcdcc1 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerConfiguration.java @@ -47,6 +47,7 @@ public class InformerConfiguration { private Set namespaces; private Boolean followControllerNamespaceChanges; private String labelSelector; + private String shardSelector; private OnAddFilter onAddFilter; private OnUpdateFilter onUpdateFilter; private OnDeleteFilter onDeleteFilter; @@ -62,6 +63,7 @@ protected InformerConfiguration( Set namespaces, boolean followControllerNamespaceChanges, String labelSelector, + String shardSelector, OnAddFilter onAddFilter, OnUpdateFilter onUpdateFilter, OnDeleteFilter onDeleteFilter, @@ -77,6 +79,7 @@ protected InformerConfiguration( this.namespaces = namespaces; this.followControllerNamespaceChanges = followControllerNamespaceChanges; this.labelSelector = labelSelector; + this.shardSelector = shardSelector; this.onAddFilter = onAddFilter; this.onUpdateFilter = onUpdateFilter; this.onDeleteFilter = onDeleteFilter; @@ -113,6 +116,7 @@ public static InformerConfiguration.Builder builder( original.namespaces, original.followControllerNamespaceChanges, original.labelSelector, + original.shardSelector, original.onAddFilter, original.onUpdateFilter, original.onDeleteFilter, @@ -125,11 +129,6 @@ public static InformerConfiguration.Builder builder( .builder; } - public static String ensureValidLabelSelector(String labelSelector) { - // might want to implement validation here? - return labelSelector; - } - public static boolean allNamespacesWatched(Set namespaces) { failIfNotValid(namespaces); return DEFAULT_NAMESPACES_SET.equals(namespaces); @@ -251,6 +250,20 @@ public String getLabelSelector() { return labelSelector; } + /** + * Retrieves the shard selector that is used, in addition to the {@link #getLabelSelector() label + * selector}, to restrict which resources are actually watched by the associated informer. + * Typically used to assign a subset (shard) of the resources to a given operator instance. It is + * expressed using the same syntax as a label selector. See the official documentation on the topic for + * more details on syntax. + * + * @return the shard selector filtering watched resources + */ + public String getShardSelector() { + return shardSelector; + } + public OnAddFilter getOnAddFilter() { return onAddFilter; } @@ -353,6 +366,11 @@ public InformerConfiguration.Builder initFromAnnotation( var labelSelector = Constants.NO_VALUE_SET.equals(fromAnnotation) ? null : fromAnnotation; withLabelSelector(labelSelector); + final var shardFromAnnotation = informerConfig.shardSelector(); + var shardSelector = + Constants.NO_VALUE_SET.equals(shardFromAnnotation) ? null : shardFromAnnotation; + withShardSelector(shardSelector); + withOnAddFilter( Utils.instantiate(informerConfig.onAddFilter(), OnAddFilter.class, context)); @@ -442,7 +460,12 @@ public Builder withFollowControllerNamespacesChanges(boolean followChanges) { } public Builder withLabelSelector(String labelSelector) { - InformerConfiguration.this.labelSelector = ensureValidLabelSelector(labelSelector); + InformerConfiguration.this.labelSelector = labelSelector; + return this; + } + + public Builder withShardSelector(String shardSelector) { + InformerConfiguration.this.shardSelector = shardSelector; return this; } diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java index 1a1d8956fc..ab1ad2b8eb 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/api/config/informer/InformerEventSourceConfiguration.java @@ -251,6 +251,11 @@ public Builder withLabelSelector(String labelSelector) { return this; } + public Builder withShardSelector(String shardSelector) { + config.withShardSelector(shardSelector); + return this; + } + public Builder withOnAddFilter(OnAddFilter onAddFilter) { config.withOnAddFilter(onAddFilter); return this; @@ -308,6 +313,7 @@ public void updateFrom(InformerConfiguration informerConfig) { .withFollowControllerNamespacesChanges( informerConfig.getFollowControllerNamespaceChanges()) .withLabelSelector(informerConfig.getLabelSelector()) + .withShardSelector(informerConfig.getShardSelector()) .withItemStore(informerConfig.getItemStore()) .withOnAddFilter(informerConfig.getOnAddFilter()) .withOnUpdateFilter(informerConfig.getOnUpdateFilter()) diff --git a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java index bfbe17c7c8..8e7054b231 100644 --- a/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java +++ b/operator-framework-core/src/main/java/io/javaoperatorsdk/operator/processing/event/source/informer/InformerManager.java @@ -136,13 +136,18 @@ public void changeNamespaces(Set namespaces) { private InformerWrapper createEventSourceForNamespace(String namespace) { final InformerWrapper source; final var labelSelector = configuration.getInformerConfig().getLabelSelector(); + final var shardSelector = configuration.getInformerConfig().getShardSelector(); if (namespace.equals(WATCH_ALL_NAMESPACES)) { - final var filteredBySelectorClient = client.inAnyNamespace().withLabelSelector(labelSelector); + final var filteredBySelectorClient = + client.inAnyNamespace().withLabelSelector(labelSelector).withShardSelector(shardSelector); source = createEventSource(filteredBySelectorClient, eventHandler, WATCH_ALL_NAMESPACES); } else { source = createEventSource( - client.inNamespace(namespace).withLabelSelector(labelSelector), + client + .inNamespace(namespace) + .withLabelSelector(labelSelector) + .withShardSelector(shardSelector), eventHandler, namespace); } @@ -275,12 +280,14 @@ public List byIndex(String indexName, String indexKey) { @Override public String toString() { final var informerConfig = configuration.getInformerConfig(); - final var selector = informerConfig.getLabelSelector(); + final var labelSelector = informerConfig.getLabelSelector(); + final var shardSelector = informerConfig.getShardSelector(); return "InformerManager [" + ReconcilerUtilsInternal.getResourceTypeNameWithVersion(configuration.getResourceClass()) + "] watching: " + informerConfig.getEffectiveNamespaces(controllerConfiguration) - + (selector != null ? " selector: " + selector : ""); + + (labelSelector != null ? " label selector: " + labelSelector : "") + + (shardSelector != null ? " shard selector: " + shardSelector : ""); } public Map informerHealthIndicators() { diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/MockKubernetesClient.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/MockKubernetesClient.java index 0000429c20..61b434c0c4 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/MockKubernetesClient.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/MockKubernetesClient.java @@ -84,6 +84,7 @@ public static KubernetesClient client( when(nonNamespaceOperation.withLabelSelector(nullable(String.class))).thenReturn(filterable); when(resources.inAnyNamespace()).thenReturn(inAnyNamespace); when(inAnyNamespace.withLabelSelector(nullable(String.class))).thenReturn(filterable); + when(filterable.withShardSelector(nullable(String.class))).thenReturn(filterable); SharedIndexInformer informer = mock(SharedIndexInformer.class); CompletableFuture informerStartRes = new CompletableFuture<>(); informerStartRes.complete(null); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java index 06ea65803d..86b8de441b 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/ControllerConfigurationOverriderTest.java @@ -181,6 +181,20 @@ void itemStorePreserved() { assertNotNull(configuration.getInformerConfig().getItemStore()); } + @Test + void shardSelectorShouldBePropagated() { + var configuration = createConfiguration(new WatchCurrentReconciler()); + assertNull(configuration.getInformerConfig().getShardSelector()); + + final var shardSelector = "shard=1"; + configuration = + ControllerConfigurationOverrider.override(configuration) + .withShardSelector(shardSelector) + .build(); + + assertEquals(shardSelector, configuration.getInformerConfig().getShardSelector()); + } + @Test void configuredDependentShouldNotChangeOnParentOverrideEvenWhenInitialConfigIsSame() { var configuration = createConfiguration(new OverriddenNSOnDepReconciler()); diff --git a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/InformerConfigurationTest.java b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/InformerConfigurationTest.java index 2631a1af82..95b8465706 100644 --- a/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/InformerConfigurationTest.java +++ b/operator-framework-core/src/test/java/io/javaoperatorsdk/operator/api/config/InformerConfigurationTest.java @@ -73,6 +73,19 @@ void nullLabelSelectorByDefault() { assertNull(informerConfig.getLabelSelector()); } + @Test + void nullShardSelectorByDefault() { + final var informerConfig = InformerConfiguration.builder(ConfigMap.class).build(); + assertNull(informerConfig.getShardSelector()); + } + + @Test + void shardSelectorIsSetOnBuilder() { + final var informerConfig = + InformerConfiguration.builder(ConfigMap.class).withShardSelector("shard=1").build(); + assertEquals("shard=1", informerConfig.getShardSelector()); + } + @Test void shouldWatchAllNamespacesByDefaultForControllers() { final var informerConfig = InformerConfiguration.builder(ConfigMap.class).buildForController(); diff --git a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java index d66b9139d4..a5b798190f 100644 --- a/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java +++ b/operator-framework/src/main/java/io/javaoperatorsdk/operator/config/loader/ConfigLoader.java @@ -143,6 +143,8 @@ public static ConfigLoader getDefault() { ControllerConfigurationOverrider::withGenerationAware), new ConfigBinding<>( "label-selector", String.class, ControllerConfigurationOverrider::withLabelSelector), + new ConfigBinding<>( + "shard-selector", String.class, ControllerConfigurationOverrider::withShardSelector), new ConfigBinding<>( "max-reconciliation-interval", Duration.class, @@ -157,6 +159,10 @@ public static ConfigLoader getDefault() { "informer.label-selector", String.class, ControllerConfigurationOverrider::withLabelSelector), + new ConfigBinding<>( + "informer.shard-selector", + String.class, + ControllerConfigurationOverrider::withShardSelector), new ConfigBinding<>( "informer.list-limit", Long.class, diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java index 1144e1c4f3..fedaf81eb6 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/config/loader/ConfigLoaderTest.java @@ -198,10 +198,12 @@ public Optional getValue(String key, Class type) { "josdk.controller.ctrl.finalizer", "josdk.controller.ctrl.generation-aware", "josdk.controller.ctrl.label-selector", + "josdk.controller.ctrl.shard-selector", "josdk.controller.ctrl.max-reconciliation-interval", "josdk.controller.ctrl.field-manager", "josdk.controller.ctrl.trigger-reconciler-on-all-events", "josdk.controller.ctrl.informer.label-selector", + "josdk.controller.ctrl.informer.shard-selector", "josdk.controller.ctrl.informer.list-limit", "josdk.controller.ctrl.rate-limiter.refresh-period", "josdk.controller.ctrl.rate-limiter.limit-for-period");