Skip to content

Commit 607e4de

Browse files
sergeytimoshinsergeytimoshin
authored
feat: add metrics for proof generation and queue wait times (#1913)
1 parent 9ac79e4 commit 607e4de

File tree

6 files changed

+210
-14
lines changed

6 files changed

+210
-14
lines changed

‎prover/server/Dockerfile.light

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM golang:1.20.3-alpine AS builder
1+
FROM golang:1.23-alpine AS builder
22

33
WORKDIR /app
44

‎prover/server/go.mod

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,42 @@
11
module light/light-prover
22

3-
go 1.21
3+
go 1.23.0
4+
5+
toolchain go1.24.4
46

57
require (
68
github.com/consensys/gnark v0.8.0
9+
github.com/google/uuid v1.6.0
710
github.com/gorilla/handlers v1.5.2
811
github.com/iden3/go-iden3-crypto v0.0.13
12+
github.com/prometheus/client_golang v1.23.0
13+
github.com/redis/go-redis/v9 v9.8.0
914
github.com/reilabs/gnark-lean-extractor/v2 v2.5.0-0.8.0
1015
github.com/urfave/cli/v2 v2.10.2
1116
)
1217

1318
require (
19+
github.com/beorn7/perks v1.0.1 // indirect
1420
github.com/blang/semver/v4 v4.0.0 // indirect
1521
github.com/cespare/xxhash/v2 v2.3.0 // indirect
1622
github.com/consensys/bavard v0.1.13 // indirect
1723
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
1824
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
1925
github.com/felixge/httpsnoop v1.0.3 // indirect
20-
github.com/google/go-cmp v0.6.0 // indirect
2126
github.com/google/pprof v0.0.0-20230309165930-d61513b1440d // indirect
22-
github.com/google/uuid v1.6.0 // indirect
2327
github.com/mattn/go-colorable v0.1.13 // indirect
2428
github.com/mattn/go-isatty v0.0.16 // indirect
2529
github.com/mitchellh/copystructure v1.2.0 // indirect
2630
github.com/mitchellh/reflectwalk v1.0.2 // indirect
27-
github.com/redis/go-redis/v9 v9.8.0 // indirect
31+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
32+
github.com/prometheus/client_model v0.6.2 // indirect
33+
github.com/prometheus/common v0.65.0 // indirect
34+
github.com/prometheus/procfs v0.16.1 // indirect
2835
github.com/rogpeppe/go-internal v1.11.0 // indirect
2936
github.com/russross/blackfriday/v2 v2.1.0 // indirect
3037
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
3138
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
39+
google.golang.org/protobuf v1.36.6 // indirect
3240
rsc.io/tmplfunc v0.0.3 // indirect
3341
)
3442

@@ -39,8 +47,8 @@ require (
3947
github.com/mmcloughlin/addchain v0.4.0 // indirect
4048
github.com/pmezard/go-difflib v1.0.0 // indirect
4149
github.com/rs/zerolog v1.29.0
42-
github.com/stretchr/testify v1.8.4
50+
github.com/stretchr/testify v1.10.0
4351
github.com/x448/float16 v0.8.4 // indirect
44-
golang.org/x/sys v0.20.0 // indirect
52+
golang.org/x/sys v0.33.0 // indirect
4553
gopkg.in/yaml.v3 v3.0.1 // indirect
4654
)

‎prover/server/go.sum

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
1+
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
2+
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
13
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
24
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
5+
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
6+
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
7+
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
8+
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
39
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
410
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
511
github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ=
@@ -22,8 +28,8 @@ github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw
2228
github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=
2329
github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
2430
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
25-
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
26-
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
31+
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
32+
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
2733
github.com/google/pprof v0.0.0-20230309165930-d61513b1440d h1:um9/pc7tKMINFfP1eE7Wv6PRGXlcCSJkVajF7KJw3uQ=
2834
github.com/google/pprof v0.0.0-20230309165930-d61513b1440d/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk=
2935
github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
@@ -33,10 +39,14 @@ github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyE
3339
github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
3440
github.com/iden3/go-iden3-crypto v0.0.13 h1:ixWRiaqDULNyIDdOWz2QQJG5t4PpNHkQk2P6GV94cok=
3541
github.com/iden3/go-iden3-crypto v0.0.13/go.mod h1:swXIv0HFbJKobbQBtsB50G7IHr6PbTowutSew/iBEoo=
42+
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
43+
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
3644
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
3745
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
3846
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
3947
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
48+
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
49+
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
4050
github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
4151
github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
4252
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
@@ -52,9 +62,19 @@ github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
5262
github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY=
5363
github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU=
5464
github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU=
65+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
66+
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
5567
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
5668
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
5769
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
70+
github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc=
71+
github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE=
72+
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
73+
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
74+
github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE=
75+
github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8=
76+
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
77+
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
5878
github.com/redis/go-redis/v9 v9.8.0 h1:q3nRvjrlge/6UD7eTu/DSg2uYiU2mCL0G/uzBWqhicI=
5979
github.com/redis/go-redis/v9 v9.8.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw=
6080
github.com/reilabs/gnark-lean-extractor/v2 v2.5.0-0.8.0 h1:2PbLqRADdX9RUqtJzacu9foC54HJ4s49/kYHyEkn6Fk=
@@ -68,14 +88,16 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf
6888
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
6989
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
7090
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
71-
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
72-
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
91+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
92+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
7393
github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y=
7494
github.com/urfave/cli/v2 v2.10.2/go.mod h1:f8iq5LtQ/bLxafbdBSLPPNsgaW0l/2fYYEHhAyPlwvo=
7595
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
7696
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
7797
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
7898
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
99+
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
100+
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
79101
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
80102
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
81103
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
@@ -87,11 +109,13 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
87109
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
88110
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
89111
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
90-
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
91-
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
112+
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
113+
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
92114
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
93115
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
94116
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
117+
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
118+
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
95119
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
96120
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
97121
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

‎prover/server/server/metrics.go

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package server
2+
3+
import (
4+
"time"
5+
6+
"github.com/prometheus/client_golang/prometheus"
7+
"github.com/prometheus/client_golang/prometheus/promauto"
8+
)
9+
10+
var (
11+
ProofRequestsTotal = promauto.NewCounterVec(
12+
prometheus.CounterOpts{
13+
Name: "prover_proof_requests_total",
14+
Help: "Total number of proof generation requests by circuit type",
15+
},
16+
[]string{"circuit_type"},
17+
)
18+
19+
ProofGenerationDuration = promauto.NewHistogramVec(
20+
prometheus.HistogramOpts{
21+
Name: "prover_proof_generation_duration_seconds",
22+
Help: "Duration of proof generation in seconds",
23+
Buckets: prometheus.ExponentialBuckets(0.1, 2, 15),
24+
},
25+
[]string{"circuit_type"},
26+
)
27+
28+
ProofGenerationErrors = promauto.NewCounterVec(
29+
prometheus.CounterOpts{
30+
Name: "prover_proof_generation_errors_total",
31+
Help: "Total number of proof generation errors by circuit type",
32+
},
33+
[]string{"circuit_type", "error_type"},
34+
)
35+
36+
QueueWaitTime = promauto.NewHistogramVec(
37+
prometheus.HistogramOpts{
38+
Name: "prover_queue_wait_time_seconds",
39+
Help: "Time spent waiting in queue before processing",
40+
Buckets: prometheus.ExponentialBuckets(0.1, 2, 12),
41+
},
42+
[]string{"circuit_type"},
43+
)
44+
45+
JobsProcessed = promauto.NewCounterVec(
46+
prometheus.CounterOpts{
47+
Name: "prover_jobs_processed_total",
48+
Help: "Total number of jobs processed",
49+
},
50+
[]string{"status"},
51+
)
52+
53+
ActiveJobs = promauto.NewGauge(
54+
prometheus.GaugeOpts{
55+
Name: "prover_active_jobs",
56+
Help: "Number of currently active proof generation jobs",
57+
},
58+
)
59+
60+
CircuitInputSize = promauto.NewHistogramVec(
61+
prometheus.HistogramOpts{
62+
Name: "prover_circuit_input_size_bytes",
63+
Help: "Size of circuit inputs in bytes",
64+
Buckets: prometheus.ExponentialBuckets(1024, 2, 15),
65+
},
66+
[]string{"circuit_type"},
67+
)
68+
69+
CircuitProofSize = promauto.NewHistogramVec(
70+
prometheus.HistogramOpts{
71+
Name: "prover_circuit_proof_size_bytes",
72+
Help: "Size of generated proofs in bytes",
73+
Buckets: prometheus.ExponentialBuckets(256, 2, 10),
74+
},
75+
[]string{"circuit_type"},
76+
)
77+
)
78+
79+
type MetricTimer struct {
80+
start time.Time
81+
circuitType string
82+
}
83+
84+
func StartProofTimer(circuitType string) *MetricTimer {
85+
ProofRequestsTotal.WithLabelValues(circuitType).Inc()
86+
ActiveJobs.Inc()
87+
return &MetricTimer{
88+
start: time.Now(),
89+
circuitType: circuitType,
90+
}
91+
}
92+
93+
func (t *MetricTimer) ObserveDuration() {
94+
duration := time.Since(t.start).Seconds()
95+
ProofGenerationDuration.WithLabelValues(t.circuitType).Observe(duration)
96+
ActiveJobs.Dec()
97+
}
98+
99+
func (t *MetricTimer) ObserveError(errorType string) {
100+
ProofGenerationErrors.WithLabelValues(t.circuitType, errorType).Inc()
101+
ActiveJobs.Dec()
102+
}
103+
104+
func RecordJobComplete(success bool) {
105+
if success {
106+
JobsProcessed.WithLabelValues("completed").Inc()
107+
} else {
108+
JobsProcessed.WithLabelValues("failed").Inc()
109+
}
110+
}
111+
112+
func RecordCircuitInputSize(circuitType string, sizeBytes int) {
113+
CircuitInputSize.WithLabelValues(circuitType).Observe(float64(sizeBytes))
114+
}
115+
116+
func RecordProofSize(circuitType string, sizeBytes int) {
117+
CircuitProofSize.WithLabelValues(circuitType).Observe(float64(sizeBytes))
118+
}

‎prover/server/server/queue_job.go

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,19 @@ func (w *BaseQueueWorker) processJobs() {
112112
return
113113
}
114114

115+
if !job.CreatedAt.IsZero() {
116+
queueWaitTime := time.Since(job.CreatedAt).Seconds()
117+
circuitType := "unknown"
118+
if w.queueName == "zk_update_queue" {
119+
circuitType = "update"
120+
} else if w.queueName == "zk_append_queue" {
121+
circuitType = "append"
122+
} else if w.queueName == "zk_address_append_queue" {
123+
circuitType = "address-append"
124+
}
125+
QueueWaitTime.WithLabelValues(circuitType).Observe(queueWaitTime)
126+
}
127+
115128
logging.Logger().Info().
116129
Str("job_id", job.ID).
117130
Str("job_type", job.Type).
@@ -170,6 +183,9 @@ func (w *BaseQueueWorker) processProofJob(job *ProofJob) error {
170183
return fmt.Errorf("failed to parse proof request: %w", err)
171184
}
172185

186+
timer := StartProofTimer(string(proofRequestMeta.CircuitType))
187+
RecordCircuitInputSize(string(proofRequestMeta.CircuitType), len(job.Payload))
188+
173189
var proof *prover.Proof
174190
var proofError error
175191

@@ -193,9 +209,19 @@ func (w *BaseQueueWorker) processProofJob(job *ProofJob) error {
193209
}
194210

195211
if proofError != nil {
212+
timer.ObserveError("proof_generation_failed")
213+
RecordJobComplete(false)
196214
return proofError
197215
}
198216

217+
timer.ObserveDuration()
218+
RecordJobComplete(true)
219+
220+
if proof != nil {
221+
proofBytes, _ := json.Marshal(proof)
222+
RecordProofSize(string(proofRequestMeta.CircuitType), len(proofBytes))
223+
}
224+
199225
resultData, _ := json.Marshal(proof)
200226
resultJob := &ProofJob{
201227
ID: job.ID,

‎prover/server/server/server.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
"github.com/redis/go-redis/v9"
1616

1717
"github.com/gorilla/handlers"
18-
//"github.com/prometheus/client_golang/prometheus/promhttp"
18+
"github.com/prometheus/client_golang/prometheus/promhttp"
1919
)
2020

2121
type proofStatusHandler struct {
@@ -460,6 +460,7 @@ func RunEnhanced(config *EnhancedConfig, redisQueue *RedisQueue, circuits []stri
460460
logging.Logger().Warn().Msg("No API key configured - server will accept all requests. Set PROVER_API_KEY environment variable to enable authentication.")
461461
}
462462
metricsMux := http.NewServeMux()
463+
metricsMux.Handle("/metrics", promhttp.Handler())
463464
metricsServer := &http.Server{Addr: config.MetricsAddress, Handler: metricsMux}
464465
metricsJob := spawnServerJob(metricsServer, "metrics server")
465466
logging.Logger().Info().Str("addr", config.MetricsAddress).Msg("metrics server started")
@@ -634,6 +635,9 @@ type healthHandler struct {
634635
func (handler proveHandler) handleAsyncProof(w http.ResponseWriter, r *http.Request, buf []byte, meta prover.ProofRequestMeta) {
635636
jobID := uuid.New().String()
636637

638+
ProofRequestsTotal.WithLabelValues(string(meta.CircuitType)).Inc()
639+
RecordCircuitInputSize(string(meta.CircuitType), len(buf))
640+
637641
job := &ProofJob{
638642
ID: jobID,
639643
Type: "zk_proof",
@@ -714,7 +718,23 @@ func (handler proveHandler) handleSyncProof(w http.ResponseWriter, r *http.Reque
714718
resultChan := make(chan proofResult, 1)
715719

716720
go func() {
721+
timer := StartProofTimer(string(meta.CircuitType))
722+
RecordCircuitInputSize(string(meta.CircuitType), len(buf))
723+
717724
proof, proofError := handler.processProofSync(buf)
725+
726+
if proofError != nil {
727+
timer.ObserveError(proofError.Code)
728+
RecordJobComplete(false)
729+
} else {
730+
timer.ObserveDuration()
731+
RecordJobComplete(true)
732+
if proof != nil {
733+
proofBytes, _ := json.Marshal(proof)
734+
RecordProofSize(string(meta.CircuitType), len(proofBytes))
735+
}
736+
}
737+
718738
resultChan <- proofResult{proof: proof, err: proofError}
719739
}()
720740

0 commit comments

Comments
 (0)