feat(api): add service tier argument for chat completions (#1486)

.stats.yml

@@ -1,2 +1,2 @@
 configured_endpoints: 64
-openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-5cb1810135c35c5024698f3365626471a04796e26e393aefe1aa0ba3c0891919.yml
+openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai-8fe357c6b5a425d810d731e4102a052d8e38c5e2d66950e6de1025415160bf88.yml

src/openai/_base_client.py

@@ -457,7 +457,7 @@ def _build_request(
  raise RuntimeError(f"Unexpected JSON data type, {type(json_data)}, cannot merge with `extra_body`")
 
  headers = self._build_headers(options)
- params = _merge_mappings(self._custom_query, options.params)
+ params = _merge_mappings(self.default_query, options.params)
  content_type = headers.get("Content-Type")
 
  # If the given Content-Type header is multipart/form-data then it
@@ -593,6 +593,12 @@ def default_headers(self) -> dict[str, str | Omit]:
  **self._custom_headers,
  }
 
+ @property
+ def default_query(self) -> dict[str, object]:
+ return {
+ **self._custom_query,
+ }
+
  def _validate_headers(
  self,
  headers: Headers, # noqa: ARG002

src/openai/resources/chat/completions.py

@@ -59,6 +59,7 @@ def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
@@ -163,6 +164,16 @@ def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream: If set, partial message deltas will be sent, like in ChatGPT. Tokens will be
@@ -236,6 +247,7 @@ def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
  temperature: Optional[float] | NotGiven = NOT_GIVEN,
@@ -346,6 +358,16 @@ def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream_options: Options for streaming response. Only set this when you set `stream: true`.
@@ -412,6 +434,7 @@ def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
  temperature: Optional[float] | NotGiven = NOT_GIVEN,
@@ -522,6 +545,16 @@ def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream_options: Options for streaming response. Only set this when you set `stream: true`.
@@ -587,6 +620,7 @@ def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
@@ -620,6 +654,7 @@ def create(
  "presence_penalty": presence_penalty,
  "response_format": response_format,
  "seed": seed,
+ "service_tier": service_tier,
  "stop": stop,
  "stream": stream,
  "stream_options": stream_options,
@@ -667,6 +702,7 @@ async def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream: Optional[Literal[False]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
@@ -771,6 +807,16 @@ async def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream: If set, partial message deltas will be sent, like in ChatGPT. Tokens will be
@@ -844,6 +890,7 @@ async def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
  temperature: Optional[float] | NotGiven = NOT_GIVEN,
@@ -954,6 +1001,16 @@ async def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream_options: Options for streaming response. Only set this when you set `stream: true`.
@@ -1020,6 +1077,7 @@ async def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
  temperature: Optional[float] | NotGiven = NOT_GIVEN,
@@ -1130,6 +1188,16 @@ async def create(
  should refer to the `system_fingerprint` response parameter to monitor changes
  in the backend.
 
+ service_tier: Specifies the latency tier to use for processing the request. This parameter is
+ relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+
  stop: Up to 4 sequences where the API will stop generating further tokens.
 
  stream_options: Options for streaming response. Only set this when you set `stream: true`.
@@ -1195,6 +1263,7 @@ async def create(
  presence_penalty: Optional[float] | NotGiven = NOT_GIVEN,
  response_format: completion_create_params.ResponseFormat | NotGiven = NOT_GIVEN,
  seed: Optional[int] | NotGiven = NOT_GIVEN,
+ service_tier: Optional[Literal["auto", "default"]] | NotGiven = NOT_GIVEN,
  stop: Union[Optional[str], List[str]] | NotGiven = NOT_GIVEN,
  stream: Optional[Literal[False]] | Literal[True] | NotGiven = NOT_GIVEN,
  stream_options: Optional[ChatCompletionStreamOptionsParam] | NotGiven = NOT_GIVEN,
@@ -1228,6 +1297,7 @@ async def create(
  "presence_penalty": presence_penalty,
  "response_format": response_format,
  "seed": seed,
+ "service_tier": service_tier,
  "stop": stop,
  "stream": stream,
  "stream_options": stream_options,

src/openai/types/chat/chat_completion.py

@@ -56,6 +56,13 @@ class ChatCompletion(BaseModel):
  object: Literal["chat.completion"]
  """The object type, which is always `chat.completion`."""
 
+ service_tier: Optional[Literal["scale", "default"]] = None
+ """The service tier used for processing the request.
+
+ This field is only included if the `service_tier` parameter is specified in the
+ request.
+ """
+
  system_fingerprint: Optional[str] = None
  """This fingerprint represents the backend configuration that the model runs with.
 

src/openai/types/chat/chat_completion_chunk.py

@@ -122,6 +122,13 @@ class ChatCompletionChunk(BaseModel):
  object: Literal["chat.completion.chunk"]
  """The object type, which is always `chat.completion.chunk`."""
 
+ service_tier: Optional[Literal["scale", "default"]] = None
+ """The service tier used for processing the request.
+
+ This field is only included if the `service_tier` parameter is specified in the
+ request.
+ """
+
  system_fingerprint: Optional[str] = None
  """
  This fingerprint represents the backend configuration that the model runs with.

src/openai/types/chat/completion_create_params.py

@@ -146,6 +146,19 @@ class CompletionCreateParamsBase(TypedDict, total=False):
  in the backend.
  """
 
+ service_tier: Optional[Literal["auto", "default"]]
+ """Specifies the latency tier to use for processing the request.
+
+ This parameter is relevant for customers subscribed to the scale tier service:
+
+ - If set to 'auto', the system will utilize scale tier credits until they are
+ exhausted.
+ - If set to 'default', the request will be processed in the shared cluster.
+
+ When this parameter is set, the response body will include the `service_tier`
+ utilized.
+ """
+
  stop: Union[Optional[str], List[str]]
  """Up to 4 sequences where the API will stop generating further tokens."""
 

tests/api_resources/chat/test_completions.py

@@ -60,6 +60,7 @@ def test_method_create_with_all_params_overload_1(self, client: OpenAI) -> None:
  presence_penalty=-2,
  response_format={"type": "json_object"},
  seed=-9223372036854776000,
+ service_tier="auto",
  stop="string",
  stream=False,
  stream_options={"include_usage": True},
@@ -176,6 +177,7 @@ def test_method_create_with_all_params_overload_2(self, client: OpenAI) -> None:
  presence_penalty=-2,
  response_format={"type": "json_object"},
  seed=-9223372036854776000,
+ service_tier="auto",
  stop="string",
  stream_options={"include_usage": True},
  temperature=1,
@@ -294,6 +296,7 @@ async def test_method_create_with_all_params_overload_1(self, async_client: Asyn
  presence_penalty=-2,
  response_format={"type": "json_object"},
  seed=-9223372036854776000,
+ service_tier="auto",
  stop="string",
  stream=False,
  stream_options={"include_usage": True},
@@ -410,6 +413,7 @@ async def test_method_create_with_all_params_overload_2(self, async_client: Asyn
  presence_penalty=-2,
  response_format={"type": "json_object"},
  seed=-9223372036854776000,
+ service_tier="auto",
  stop="string",
  stream_options={"include_usage": True},
  temperature=1,