Deprequeue optimizations #1703
Open
Deprequeue optimizations #1703
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
Thanks for your pull request! It looks like this may be your first contribution to a Google open source project. Before we can look at your pull request, you'll need to sign a Contributor License Agreement (CLA). View this failed invocation of the CLA check for more information. For the most up to date status, view the checks section at the bottom of the pull request. |
avardaro
force-pushed
the
deprequeue_optimizations
branch
7 times, most recently
from
4e4f739
to
7df0ff0
Compare
werkt
requested changes
Apr 16, 2024
.bazelci/run_abseil_test.sh
Outdated
Comment on lines
3
to
4
| ../../../bazel test --jobs=25 $1 --test_output=errors --incompatible_enable_cc_toolchain_resolution --verbose_failures --verbose_explanations --test_tag_filters=-benchmark --remote_executor=grpc://localhost:8980 @com_google_absl//... -- -@com_google_absl//absl/time/... No newline at end of file | ||
| ../../../bazel test --jobs=25 $1 --test_output=errors --incompatible_enable_cc_toolchain_resolution --verbose_failures --verbose_explanations --test_tag_filters=-benchmark --remote_executor=grpc://localhost:8980 @com_google_absl//... -- -@com_google_absl//absl/time/... |
There was a problem hiding this comment.
unnecessary trailing newline add/remove
| @@ -49,6 +49,7 @@ public enum BACKPLANE_TYPE { | |||
| @Getter(AccessLevel.NONE) | |||
| private boolean runFailsafeOperation = true; // deprecated | |||
|
|
|||
| // DBG_AVA remove maxQueueDepth | |||
| @@ -46,8 +46,9 @@ | |||
| * the same underlying redis queues. | |||
| */ | |||
| public class BalancedRedisQueue { | |||
| private static final Duration START_TIMEOUT = Duration.ofSeconds(1); | |||
| public static final int UNLIMITED_QUEUE_DEPTH = -1; | |||
There was a problem hiding this comment.
Consolidate this value from the config - they shouldn't have different representations
| // We assume the number of elements in the queue is evenly distributed across the Redis nodes. | ||
| // As such, we're just going to check against against the maximum number of elements an | ||
| // individual Redis node should have rather than across all the Redis nodes. | ||
| // DBG_AVA handle the case where a maxNodeQueueDepth is 0. |
| } | ||
| return hexString.toString(); | ||
| } catch (NoSuchAlgorithmException e) { | ||
| throw new RuntimeException("SHA-1 algorithm not found", e); |
There was a problem hiding this comment.
This seems a bit funky - is there nothing in jedis to help with doing this hashing in a way that doesn't bind us to the whims of a redis impl digest function?
Comment on lines
+318
to
+294
|
|
||
| # DBG_AVA | ||
| # java_binary( | ||
| # name = "fill-prequeue", | ||
| # srcs = ["FillPrequeue.java"], | ||
| # main_class = "build.buildfarm.tools.FillPrequeue", | ||
| # visibility = ["//visibility:public"], | ||
| # deps = [ | ||
| # "//src/main/java/build/buildfarm/common/config", | ||
| # "//src/main/java/build/buildfarm/common/resources:resource_java_proto", | ||
| # "//src/main/java/build/buildfarm/instance/shard", | ||
| # "//src/main/protobuf:build_buildfarm_v1test_buildfarm_java_proto", | ||
| # "@com_google_googleapis//google/longrunning:operations_proto", | ||
| # "@maven//:com_google_guava_guava", | ||
| # "@maven//:com_google_protobuf_protobuf_java", | ||
| # "@maven//:redis_clients_jedis", | ||
| # ], | ||
| # ) |
| @@ -358,11 +358,14 @@ message ExecuteEntry { | |||
| string stderr_stream_name = 8; | |||
|
|
|||
| google.protobuf.Timestamp queued_timestamp = 9; | |||
|
|
|||
| optional build.bazel.remote.execution.v2.Digest queued_operation_digest = 10; | |||
There was a problem hiding this comment.
Remove optional - support is deprecated for it in proto3
| } | ||
|
|
||
| message QueueEntry { | ||
| ExecuteEntry execute_entry = 1; | ||
|
|
||
| // DBG_AVA reserved 2; |
Comment on lines
355
to
366
| // when(subQueue.size()).thenReturn(123L); | ||
|
|
||
| // ARRANGE | ||
| BalancedRedisQueue queue = | ||
| new BalancedRedisQueue("test", ImmutableList.of("test"), 123, this::subQueueDecorate); | ||
| // BalancedRedisQueue queue = | ||
| // new BalancedRedisQueue("test", ImmutableList.of("test"), 123, this::subQueueDecorate); | ||
|
|
||
| // ACT | ||
| boolean canQueue = queue.canQueue(redis); | ||
| // DBG_AVA replace with attempt to queue boolean canQueue = queue.canQueue(redis); | ||
|
|
||
| // ASSERT | ||
| verify(subQueue, times(1)).size(); | ||
| assertThat(canQueue).isFalse(); | ||
| // DBG_AVA verify(subQueue, times(1)).size(); | ||
| // DBG_AVA assertThat(canQueue).isFalse(); |
| /** | ||
| * @class RedisLuaScriptTest | ||
| * @brief exercises the RedisLuaScript class | ||
| * @details DBG_AVA |
avardaro
force-pushed
the
deprequeue_optimizations
branch
from
7df0ff0
to
6f13995
Compare
avardaro
force-pushed
the
deprequeue_optimizations
branch
from
6f13995
to
4a99ca4
Compare
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
NO-FLIGHT