TorchServe

コレクションでコンテンツを整理 必要に応じて、コンテンツの保存と分類を行います。

# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

  inference_address=http://0.0.0.0:8080
  management_address=http://0.0.0.0:8081
+ metrics_address=http://0.0.0.0:8082
+ metrics_mode=prometheus
  number_of_netty_threads=32
  job_queue_size=1000
  install_py_dep_per_model=true
  model_store=/home/model-server/model-store
  load_models=all

# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: apps/v1
kind: Deployment
metadata:
  name: t5-inference
  labels:
    model: t5
    version: v1.0
    machine: gpu
spec:
  replicas: 1
  selector:
    matchLabels:
      model: t5
      version: v1.0
      machine: gpu
  template:
    metadata:
      labels:
        model: t5
        version: v1.0
        machine: gpu
    spec:
      nodeSelector:
        cloud.google.com/gke-accelerator: nvidia-l4
      containers:
        - name: inference
          ...
          args: ["torchserve", "--start", "--foreground"]
          resources:
            ...
          ports:
            - containerPort: 8080
              name: http
            - containerPort: 8081
              name: management
+           - containerPort: 8082
+             name: metrics
---
apiVersion: v1
kind: Service
metadata:
  name: t5-inference
  labels:
    model: t5
    version: v1.0
    machine: gpu
spec:
  ...
  ports:
    - port: 8080
      name: http
      targetPort: http
    - port: 8081
      name: management
      targetPort: management
+   - port: 8082
+     name: metrics
+     targetPort: metrics

# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: monitoring.googleapis.com/v1
kind: PodMonitoring
metadata:
  name: torchserve
  labels:
    app.kubernetes.io/name: torchserve
    app.kubernetes.io/part-of: google-cloud-managed-prometheus
spec:
  endpoints:
  - port: 8082
    scheme: http
    interval: 30s
    path: /metrics
  selector:
    matchLabels:
      model: t5
      version: v1.0
      machine: gpu