crash when put with the same prediction id twice #2454
Description
When I send two PUT requests to the same model using the same prediction ID, the Cog server crashes 100% of the time — of course, the second request is made while the first one is still being processed.
I'am using the latest cog with version v0.15.10
The crash log as follows:
{"logger": "uvicorn.error", "timestamp": "2025-07-08T12:08:42.583893Z", "exception": "Traceback (most recent call last):\n File \"/usr/local/lib/python3.13/site-packages/uvicorn/protocols/http/httptools_impl.py\", line 409, in run_asgi\n result = await app( # type: ignore[func-returns-value]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n self.scope, self.receive, self.send\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"/usr/local/lib/python3.13/site-packages/uvicorn/middleware/proxy_headers.py\", line 60, in __call__\n return await self.app(scope, receive, send)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/site-packages/fastapi/applications.py\", line 1054, in __call__\n await super().__call__(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/applications.py\", line 112, in __call__\n await self.middleware_stack(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/middleware/errors.py\", line 187, in __call__\n raise exc\n File \"/usr/local/lib/python3.13/site-packages/starlette/middleware/errors.py\", line 165, in __call__\n await self.app(scope, receive, _send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/middleware/exceptions.py\", line 62, in __call__\n await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/usr/local/lib/python3.13/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/usr/local/lib/python3.13/site-packages/starlette/routing.py\", line 714, in __call__\n await self.middleware_stack(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/routing.py\", line 734, in app\n await route.handle(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/routing.py\", line 288, in handle\n await self.app(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/routing.py\", line 76, in app\n await wrap_app_handling_exceptions(app, request)(scope, receive, send)\n File \"/usr/local/lib/python3.13/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/usr/local/lib/python3.13/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/usr/local/lib/python3.13/site-packages/starlette/routing.py\", line 73, in app\n response = await f(request)\n ^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/site-packages/fastapi/routing.py\", line 301, in app\n raw_response = await run_endpoint_function(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n )\n ^\n File \"/usr/local/lib/python3.13/site-packages/fastapi/routing.py\", line 212, in run_endpoint_function\n return await dependant.call(**values)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/site-packages/cog/server/http.py\", line 433, in predict_idempotent\n return await _predict(\n ^^^^^^^^^^^^^^^\n ...<3 lines>...\n )\n ^\n File \"/usr/local/lib/python3.13/site-packages/cog/server/http.py\", line 464, in _predict\n predict_task = runner.predict(request, is_train, task_kwargs=task_kwargs)\n File \"/usr/local/lib/python3.13/site-packages/cog/server/runner.py\", line 118, in predict\n task.track(self._worker.predict(payload, context=prediction.context, tag=tag))\n ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.13/site-packages/cog/server/worker.py\", line 159, in predict\ncog.server.exceptions.InvalidStateException: Invalid operation: prediction with tag 0 already running", "severity": "ERROR", "message": "Exception in ASGI application\n"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:42.642989Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:42.743412Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:42.851194Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:42.940660Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.048365Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.141420Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.237999Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.342598Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.448318Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.572028Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.676047Z", "severity": "INFO", "message": "prediction failed"} {"prediction_id": "0", "error": "Predictor unexpectedly returned output before output type", "logger": "cog.server.runner", "timestamp": "2025-07-08T12:08:43.791143Z", "severity": "INFO", "message": "prediction failed"}