promql (histograms): reconcile mismatched NHCB bounds (#17278)

Fixes #17255.

The implementation happens mostly in the Add and Sub method, but the reconciliation works for all relevant operations. For example, you can now `rate` over a range wherein the custom bucket boundaries are changing.

Any custom bucket reconciliation is flagged with an info-level annotation.

---------

Signed-off-by: Linas Medziunas <linas.medziunas@gmail.com>
Signed-off-by: Linas Medžiūnas <linasm@users.noreply.github.com>

model/histogram/float_histogram.go

@@ -343,10 +343,13 @@ func (h *FloatHistogram) Div(scalar float64) *FloatHistogram {
 // is returned as true. A counter reset conflict occurs iff one of two histograms indicate
 // a counter reset (CounterReset) while the other indicates no reset (NotCounterReset).
 //
+// In case of mismatched NHCB bounds, they will be reconciled to the intersection of
+// both histograms, and nhcbBoundsReconciled will be returned as true.
+//
 // This method returns a pointer to the receiving histogram for convenience.
-func (h *FloatHistogram) Add(other *FloatHistogram) (res *FloatHistogram, counterResetCollision bool, err error) {
+func (h *FloatHistogram) Add(other *FloatHistogram) (res *FloatHistogram, counterResetCollision, nhcbBoundsReconciled bool, err error) {
 	if err := h.checkSchemaAndBounds(other); err != nil {
-		return nil, false, err
+		return nil, false, false, err
 	}
 	counterResetCollision = h.adjustCounterReset(other)
 	if !h.UsesCustomBuckets() {
@@ -364,8 +367,21 @@ func (h *FloatHistogram) Add(other *FloatHistogram) (res *FloatHistogram, counte
 	)
 
 	if h.UsesCustomBuckets() {
-		h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
-		return h, counterResetCollision, nil
+		if CustomBucketBoundsMatch(h.CustomValues, other.CustomValues) {
+			h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
+		} else {
+			nhcbBoundsReconciled = true
+			intersectedBounds := intersectCustomBucketBounds(h.CustomValues, other.CustomValues)
+
+			// Add with mapping - maps both histograms to intersected layout.
+			h.PositiveSpans, h.PositiveBuckets = addCustomBucketsWithMismatches(
+				false,
+				hPositiveSpans, hPositiveBuckets, h.CustomValues,
+				otherPositiveSpans, otherPositiveBuckets, other.CustomValues,
+				intersectedBounds)
+			h.CustomValues = intersectedBounds
+		}
+		return h, counterResetCollision, nhcbBoundsReconciled, nil
 	}
 
 	var (
@@ -389,7 +405,7 @@ func (h *FloatHistogram) Add(other *FloatHistogram) (res *FloatHistogram, counte
 	h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
 	h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, false, hNegativeSpans, hNegativeBuckets, otherNegativeSpans, otherNegativeBuckets)
 
-	return h, counterResetCollision, nil
+	return h, counterResetCollision, nhcbBoundsReconciled, nil
 }
 
 // Sub works like Add but subtracts the other histogram. It uses the same logic
@@ -397,9 +413,9 @@ func (h *FloatHistogram) Add(other *FloatHistogram) (res *FloatHistogram, counte
 // for incremental mean calculation. However, if it is used for the actual "-"
 // operator in PromQL, the counter reset needs to be set to GaugeType after
 // calling this method.
-func (h *FloatHistogram) Sub(other *FloatHistogram) (res *FloatHistogram, counterResetCollision bool, err error) {
+func (h *FloatHistogram) Sub(other *FloatHistogram) (res *FloatHistogram, counterResetCollision, nhcbBoundsReconciled bool, err error) {
 	if err := h.checkSchemaAndBounds(other); err != nil {
-		return nil, false, err
+		return nil, false, false, err
 	}
 	counterResetCollision = h.adjustCounterReset(other)
 	if !h.UsesCustomBuckets() {
@@ -417,8 +433,21 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) (res *FloatHistogram, counte
 	)
 
 	if h.UsesCustomBuckets() {
-		h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
-		return h, counterResetCollision, nil
+		if CustomBucketBoundsMatch(h.CustomValues, other.CustomValues) {
+			h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
+		} else {
+			nhcbBoundsReconciled = true
+			intersectedBounds := intersectCustomBucketBounds(h.CustomValues, other.CustomValues)
+
+			// Subtract with mapping - maps both histograms to intersected layout.
+			h.PositiveSpans, h.PositiveBuckets = addCustomBucketsWithMismatches(
+				true,
+				hPositiveSpans, hPositiveBuckets, h.CustomValues,
+				otherPositiveSpans, otherPositiveBuckets, other.CustomValues,
+				intersectedBounds)
+			h.CustomValues = intersectedBounds
+		}
+		return h, counterResetCollision, nhcbBoundsReconciled, nil
 	}
 
 	var (
@@ -441,7 +470,7 @@ func (h *FloatHistogram) Sub(other *FloatHistogram) (res *FloatHistogram, counte
 	h.PositiveSpans, h.PositiveBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hPositiveSpans, hPositiveBuckets, otherPositiveSpans, otherPositiveBuckets)
 	h.NegativeSpans, h.NegativeBuckets = addBuckets(h.Schema, h.ZeroThreshold, true, hNegativeSpans, hNegativeBuckets, otherNegativeSpans, otherNegativeBuckets)
 
-	return h, counterResetCollision, nil
+	return h, counterResetCollision, nhcbBoundsReconciled, nil
 }
 
 // Equals returns true if the given float histogram matches exactly.
@@ -604,11 +633,17 @@ func (h *FloatHistogram) DetectReset(previous *FloatHistogram) bool {
 	if h.Count < previous.Count {
 		return true
 	}
-	if h.UsesCustomBuckets() != previous.UsesCustomBuckets() || (h.UsesCustomBuckets() && !CustomBucketBoundsMatch(h.CustomValues, previous.CustomValues)) {
-		// Mark that something has changed or that the application has been restarted. However, this does
-		// not matter so much since the change in schema will be handled directly in the chunks and PromQL
-		// functions.
-		return true
+	if h.UsesCustomBuckets() {
+		if !previous.UsesCustomBuckets() {
+			// Mark that something has changed or that the application has been restarted. However, this does
+			// not matter so much since the change in schema will be handled directly in the chunks and PromQL
+			// functions.
+			return true
+		}
+		if !CustomBucketBoundsMatch(h.CustomValues, previous.CustomValues) {
+			// Custom bounds don't match - check if any reconciled bucket value has decreased.
+			return h.detectResetWithMismatchedCustomBounds(previous, h.CustomValues, previous.CustomValues)
+		}
 	}
 	if h.Schema > previous.Schema {
 		return true
@@ -1331,6 +1366,204 @@ func floatBucketsMatch(b1, b2 []float64) bool {
 	return true
 }
 
+// detectResetWithMismatchedCustomBounds checks if any bucket count has decreased when
+// comparing NHCBs with mismatched custom bounds. It maps both histograms
+// to the intersected bounds on-the-fly and compares values without allocating
+// arrays for all mapped buckets.
+// Will panic if called with histograms that are not NHCB.
+func (h *FloatHistogram) detectResetWithMismatchedCustomBounds(
+	previous *FloatHistogram, currBounds, prevBounds []float64,
+) bool {
+	if h.Schema != CustomBucketsSchema || previous.Schema != CustomBucketsSchema {
+		panic("detectResetWithMismatchedCustomBounds called with non-NHCB schema")
+	}
+	currIt := h.floatBucketIterator(true, 0, CustomBucketsSchema)
+	prevIt := previous.floatBucketIterator(true, 0, CustomBucketsSchema)
+
+	rollupSumForBound := func(iter *floatBucketIterator, iterStarted bool, iterBucket Bucket[float64], bound float64) (float64, Bucket[float64], bool) {
+		if !iterStarted {
+			if !iter.Next() {
+				return 0, Bucket[float64]{}, false
+			}
+			iterBucket = iter.At()
+		}
+		var sum float64
+		for iterBucket.Upper <= bound {
+			sum += iterBucket.Count
+			if !iter.Next() {
+				return sum, Bucket[float64]{}, false
+			}
+			iterBucket = iter.At()
+		}
+		return sum, iterBucket, true
+	}
+
+	var (
+		currBoundIdx, prevBoundIdx   = 0, 0
+		currBucket, prevBucket       Bucket[float64]
+		currIterStarted, currHasMore bool
+		prevIterStarted, prevHasMore bool
+	)
+
+	for currBoundIdx <= len(currBounds) && prevBoundIdx <= len(prevBounds) {
+		currBound := math.Inf(1)
+		if currBoundIdx < len(currBounds) {
+			currBound = currBounds[currBoundIdx]
+		}
+		prevBound := math.Inf(1)
+		if prevBoundIdx < len(prevBounds) {
+			prevBound = prevBounds[prevBoundIdx]
+		}
+
+		switch {
+		case currBound == prevBound:
+			// Check matching bound, rolling up lesser buckets that have not been accounter for yet.
+			currRollupSum := 0.0
+			if !currIterStarted || currHasMore {
+				currRollupSum, currBucket, currHasMore = rollupSumForBound(&currIt, currIterStarted, currBucket, currBound)
+				currIterStarted = true
+			}
+
+			prevRollupSum := 0.0
+			if !prevIterStarted || prevHasMore {
+				prevRollupSum, prevBucket, prevHasMore = rollupSumForBound(&prevIt, prevIterStarted, prevBucket, currBound)
+				prevIterStarted = true
+			}
+
+			if currRollupSum < prevRollupSum {
+				return true
+			}
+
+			currBoundIdx++
+			prevBoundIdx++
+		case currBound < prevBound:
+			currBoundIdx++
+		default:
+			prevBoundIdx++
+		}
+	}
+
+	return false
+}
+
+// intersectCustomBucketBounds returns the intersection of two custom bucket boundary sets.
+func intersectCustomBucketBounds(boundsA, boundsB []float64) []float64 {
+	if len(boundsA) == 0 || len(boundsB) == 0 {
+		return nil
+	}
+
+	var (
+		result []float64
+		i, j   = 0, 0
+	)
+
+	for i < len(boundsA) && j < len(boundsB) {
+		switch {
+		case boundsA[i] == boundsB[j]:
+			if result == nil {
+				// Allocate a new slice because FloatHistogram.CustomValues has to be immutable.
+				result = make([]float64, 0, min(len(boundsA), len(boundsB)))
+			}
+			result = append(result, boundsA[i])
+			i++
+			j++
+		case boundsA[i] < boundsB[j]:
+			i++
+		default:
+			j++
+		}
+	}
+
+	return result
+}
+
+// addCustomBucketsWithMismatches handles adding/subtracting custom bucket histograms
+// with mismatched bucket layouts by mapping both to an intersected layout.
+func addCustomBucketsWithMismatches(
+	negative bool,
+	spansA []Span, bucketsA, boundsA []float64,
+	spansB []Span, bucketsB, boundsB []float64,
+	intersectedBounds []float64,
+) ([]Span, []float64) {
+	targetBuckets := make([]float64, len(intersectedBounds)+1)
+
+	mapBuckets := func(spans []Span, buckets, bounds []float64, negative bool) {
+		srcIdx := 0
+		bucketIdx := 0
+		intersectIdx := 0
+
+		for _, span := range spans {
+			srcIdx += int(span.Offset)
+			for range span.Length {
+				if bucketIdx < len(buckets) {
+					value := buckets[bucketIdx]
+
+					// Find target bucket index.
+					targetIdx := len(targetBuckets) - 1 // Default to +Inf bucket.
+					if srcIdx < len(bounds) {
+						srcBound := bounds[srcIdx]
+						// Since both arrays are sorted, we can continue from where we left off.
+						for intersectIdx < len(intersectedBounds) {
+							if intersectedBounds[intersectIdx] >= srcBound {
+								targetIdx = intersectIdx
+								break
+							}
+							intersectIdx++
+						}
+					}
+
+					if negative {
+						targetBuckets[targetIdx] -= value
+					} else {
+						targetBuckets[targetIdx] += value
+					}
+				}
+				srcIdx++
+				bucketIdx++
+			}
+		}
+	}
+
+	// Map both histograms to the intersected layout.
+	mapBuckets(spansA, bucketsA, boundsA, false)
+	mapBuckets(spansB, bucketsB, boundsB, negative)
+
+	// Build spans and buckets, excluding zero-valued buckets from the final result.
+	destSpans := spansA[:0]          // Reuse spansA capacity for destSpans since we don't need it anymore.
+	destBuckets := targetBuckets[:0] // Reuse targetBuckets capacity for destBuckets since it's guaranteed to be large enough.
+	lastIdx := int32(-1)
+
+	for i, count := range targetBuckets {
+		if count == 0 {
+			continue
+		}
+
+		destBuckets = append(destBuckets, count)
+		idx := int32(i)
+
+		if len(destSpans) > 0 && idx == lastIdx+1 {
+			// Consecutive bucket, extend the last span.
+			destSpans[len(destSpans)-1].Length++
+		} else {
+			// New span needed.
+			// TODO: optimize away small gaps.
+			offset := idx
+			if len(destSpans) > 0 {
+				// Convert to relative offset from the end of the last span.
+				prevEnd := lastIdx
+				offset = idx - prevEnd - 1
+			}
+			destSpans = append(destSpans, Span{
+				Offset: offset,
+				Length: 1,
+			})
+		}
+		lastIdx = idx
+	}
+
+	return destSpans, destBuckets
+}
+
 // ReduceResolution reduces the float histogram's spans, buckets into target schema.
 // The target schema must be smaller than the current float histogram's schema.
 // This will panic if the histogram has custom buckets or if the target schema is
@@ -1354,15 +1587,11 @@ func (h *FloatHistogram) ReduceResolution(targetSchema int32) *FloatHistogram {
 }
 
 // checkSchemaAndBounds checks if two histograms are compatible because they
-// both use a standard exponential schema or because they both are NHCBs. In the
-// latter case, they also have to use the same custom bounds.
+// both use a standard exponential schema or because they both are NHCBs.
 func (h *FloatHistogram) checkSchemaAndBounds(other *FloatHistogram) error {
 	if h.UsesCustomBuckets() != other.UsesCustomBuckets() {
 		return ErrHistogramsIncompatibleSchema
 	}
-	if h.UsesCustomBuckets() && !CustomBucketBoundsMatch(h.CustomValues, other.CustomValues) {
-		return ErrHistogramsIncompatibleBounds
-	}
 	return nil
 }
 

model/histogram/float_histogram_test.go

@@ -1275,6 +1275,146 @@ func TestFloatHistogramDetectReset(t *testing.T) {
 			},
 			true,
 		},
+		{
+			"mismatched custom bounds - no reset when all buckets increase",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 4}},
+				PositiveBuckets: []float64{20, 35, 40, 50}, // Previous: buckets for: (-Inf,0], (0,1], (1,2], (2,3], then (3,+Inf]
+				CustomValues:    []float64{0, 1, 2, 3},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 5}},
+				PositiveBuckets: []float64{25, 15, 40, 50, 70}, // Current: buckets for: (-Inf,0], (0,0.5], (0.5,1], (1,2], (2,3], then (3,+Inf]
+				CustomValues:    []float64{0, 0.5, 1, 2, 3},
+			},
+			false, // No reset: (-Inf,0] increases from 20 to 25, (0,1] increases from 35 to 55, (1,2] increases from 40 to 50, (2,3] increases from 50 to 70
+		},
+		{
+			"mismatched custom bounds - reset in middle bucket",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 4}},
+				PositiveBuckets: []float64{10, 15, 20, 25}, // Buckets for: [0,1], [1,3], [3,5], [5,+Inf]
+				CustomValues:    []float64{0, 1, 3, 5},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 5}},
+				PositiveBuckets: []float64{10, 16, 10, 5, 25}, // Buckets for: [0,1], [1,2], [2,3], [3,5], [5,+Inf]
+				CustomValues:    []float64{0, 1, 2, 3, 5},
+			},
+			true, // Reset detected: [1,3] bucket decreased from 20 to 15
+		},
+		{
+			"mismatched custom bounds - reset in last bucket",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 3}},
+				PositiveBuckets: []float64{10, 20, 20}, // Buckets for: [0,1], [1,2], [2,+Inf]
+				CustomValues:    []float64{0, 1, 2},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 4}},
+				PositiveBuckets: []float64{100, 200, 8, 7}, // Buckets for: [0,1], [1,2], [2,3], [3,+Inf]
+				CustomValues:    []float64{0, 1, 2, 3},
+			},
+			true, // Reset detected: [2,+Inf] bucket decreased from 20 to 15
+		},
+		{
+			"mismatched custom bounds - no common bounds",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 3}},
+				PositiveBuckets: []float64{10, 20, 30}, // Buckets for: [1,2], [2,3], [3,+Inf]
+				CustomValues:    []float64{4, 5, 6},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           100,
+				Sum:             500,
+				PositiveSpans:   []Span{{1, 3}},
+				PositiveBuckets: []float64{15, 25, 35}, // Buckets for: [4,5], [5,6], [6,+Inf]
+				CustomValues:    []float64{1, 2, 3},
+			},
+			false, // no decrease in aggregated single +Inf bounded bucket
+		},
+		{
+			"mismatched custom bounds - sparse common bounds",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 4}},
+				PositiveBuckets: []float64{10, 20, 30, 40}, // Previous: buckets for: (-Inf,0], (0,1], (1,3], (3,5], then (5,+Inf]
+				CustomValues:    []float64{0, 1, 3, 5},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 5}},
+				PositiveBuckets: []float64{15, 25, 70, 50, 100}, // Current: buckets for: (-Inf,0], (0,2], (2,3], (3,4], (4,5], then (5,+Inf]
+				CustomValues:    []float64{0, 2, 3, 4, 5},
+			},
+			false, // No reset: common bounds [0,3,5] all increase when mapped
+		},
+		{
+			"reset detected with mismatched custom bounds and split bucket spans",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 2}, {1, 3}},        // Split spans: buckets at indices 0,1 and 3,4,5
+				PositiveBuckets: []float64{10, 20, 30, 40, 50}, // Buckets for: (-Inf,0], (0,1], skip (1,2], then (2,3], (3,4], (4,5]
+				CustomValues:    []float64{0, 1, 2, 3, 4, 5},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           200,
+				Sum:             1000,
+				PositiveSpans:   []Span{{0, 3}, {1, 2}},        // Split spans: buckets at indices 0,1,2 and 4,5
+				PositiveBuckets: []float64{15, 25, 35, 25, 60}, // Buckets for: (-Inf,0], (0,1], (1,3], skip (3,4], then (4,5], (5,7]
+				CustomValues:    []float64{0, 1, 3, 4, 5, 7},
+			},
+			true, // Reset detected: bucket (3,4] goes from 40 to 0 (missing in current histogram)
+		},
+		{
+			"no reset with mismatched custom bounds and split bucket spans",
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           300,
+				Sum:             1500,
+				PositiveSpans:   []Span{{0, 2}, {2, 3}},        // Split spans: buckets at indices 0,1 and 4,5,6
+				PositiveBuckets: []float64{10, 20, 30, 40, 50}, // Buckets for: (-Inf,0], (0,1], skip (1,2], (2,3], then (3,4], (4,5], (5,6]
+				CustomValues:    []float64{0, 1, 2, 3, 4, 5, 6},
+			},
+			&FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           300,
+				Sum:             1500,
+				PositiveSpans:   []Span{{0, 3}, {1, 2}},        // Split spans: buckets at indices 0,1,2 and 4,5
+				PositiveBuckets: []float64{12, 25, 45, 75, 95}, // Buckets for: (-Inf,0], (0,0.5], (0.5,1], skip (1,3], then (3,5], (5,7]
+				CustomValues:    []float64{0, 0.5, 1, 3, 5, 7},
+			},
+			false, // No reset: all mapped buckets increase
+		},
 	}
 
 	for _, c := range cases {
@@ -1649,6 +1789,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 		in1, in2, expected       *FloatHistogram
 		expErrMsg                string
 		expCounterResetCollision bool
+		expNHCBBoundsReconciled  bool
 	}{
 		{
 			name: "same bucket layout",
@@ -2093,7 +2234,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2101,7 +2242,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{0, 0, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2109,7 +2250,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{1, 0, 5, 7, 13},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 		},
 		{
@@ -2120,7 +2261,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {1, 1}, {0, 2}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2128,7 +2269,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{0, 2}, {1, 2}, {0, 1}},
 				PositiveBuckets: []float64{0, 0, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2136,7 +2277,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 2}, {1, 1}, {0, 2}},
 				PositiveBuckets: []float64{1, 0, 5, 7, 13},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 		},
 		{
@@ -2147,7 +2288,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {2, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2155,7 +2296,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{2, 2}, {3, 3}},
 				PositiveBuckets: []float64{5, 4, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2163,7 +2304,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 4}, {0, 6}},
 				PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 		},
 		{
@@ -2174,7 +2315,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{2, 2}, {3, 3}},
 				PositiveBuckets: []float64{5, 4, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2182,7 +2323,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {2, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2190,7 +2331,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 4}, {0, 6}},
 				PositiveBuckets: []float64{1, 0, 5, 4, 3, 4, 7, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7, 8, 9},
 			},
 		},
 		{
@@ -2201,7 +2342,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {2, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2209,7 +2350,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{1, 4}, {0, 3}},
 				PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2217,7 +2358,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 4}, {0, 4}},
 				PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 		},
 		{
@@ -2228,7 +2369,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             1.234,
 				PositiveSpans:   []Span{{1, 4}, {0, 3}},
 				PositiveBuckets: []float64{5, 4, 2, 3, 6, 2, 5},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2236,7 +2377,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             2.345,
 				PositiveSpans:   []Span{{0, 2}, {2, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 			expected: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2244,28 +2385,92 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             3.579,
 				PositiveSpans:   []Span{{0, 4}, {0, 4}},
 				PositiveBuckets: []float64{1, 5, 4, 2, 6, 10, 9, 5},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6, 7},
 			},
 		},
 		{
-			name: "different custom bucket layout",
+			name: "custom buckets with partial intersection",
 			in1: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
-				Count:           15,
-				Sum:             2.345,
-				PositiveSpans:   []Span{{0, 2}, {1, 3}},
-				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				Count:           10,
+				Sum:             100,
+				PositiveSpans:   []Span{{0, 3}},
+				PositiveBuckets: []float64{2, 3, 5},
+				CustomValues:    []float64{1, 2.5},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
-				Count:           11,
+				Count:           8,
+				Sum:             80,
+				PositiveSpans:   []Span{{0, 4}},
+				PositiveBuckets: []float64{1, 2, 3, 2},
+				CustomValues:    []float64{1, 2, 3},
+			},
+			expected: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           18,
+				Sum:             180,
+				PositiveSpans:   []Span{{0, 2}},
+				PositiveBuckets: []float64{3, 15},
+				CustomValues:    []float64{1},
+			},
+			expNHCBBoundsReconciled: true,
+		},
+		{
+			name: "different custom bucket layout - intersection and rollup",
+			in1: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           6,
+				Sum:             2.345,
+				PositiveSpans:   []Span{{0, 1}, {1, 1}},
+				PositiveBuckets: []float64{1, 5},
+				CustomValues:    []float64{2, 4},
+			},
+			in2: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           220,
 				Sum:             1.234,
-				PositiveSpans:   []Span{{0, 2}, {1, 3}},
-				PositiveBuckets: []float64{0, 0, 2, 3, 6},
+				PositiveSpans:   []Span{{0, 2}, {1, 1}, {0, 2}},
+				PositiveBuckets: []float64{10, 20, 40, 50, 100},
 				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
-			expErrMsg: "cannot apply this operation on custom buckets histograms with different custom bounds",
+			expected: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           226,
+				Sum:             3.579,
+				PositiveSpans:   []Span{{0, 3}},
+				PositiveBuckets: []float64{1 + 10 + 20, 40, 5 + 50 + 100},
+				CustomValues:    []float64{2, 4},
+			},
+			expNHCBBoundsReconciled: true,
+		},
+		{
+			name: "custom buckets with no common boundaries except +Inf",
+			in1: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           3,
+				Sum:             50,
+				PositiveSpans:   []Span{{0, 2}},
+				PositiveBuckets: []float64{1, 2},
+				CustomValues:    []float64{1.5},
+			},
+			in2: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           30,
+				Sum:             40,
+				PositiveSpans:   []Span{{0, 2}},
+				PositiveBuckets: []float64{10, 20},
+				CustomValues:    []float64{2.5},
+			},
+			expected: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           33,
+				Sum:             90,
+				PositiveSpans:   []Span{{0, 1}},
+				PositiveBuckets: []float64{1 + 2 + 10 + 20},
+				CustomValues:    nil,
+			},
+			expNHCBBoundsReconciled: true,
 		},
 		{
 			name: "mix exponential and custom buckets histograms",
@@ -2286,7 +2491,7 @@ func TestFloatHistogramAdd(t *testing.T) {
 				Sum:             12,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{0, 0, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			expErrMsg: "cannot apply this operation on histograms with a mix of exponential and custom bucket schemas",
 		},
@@ -2307,13 +2512,16 @@ func TestFloatHistogramAdd(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			testHistogramAdd(t, c.in1, c.in2, c.expected, c.expErrMsg, c.expCounterResetCollision)
-			testHistogramAdd(t, c.in2, c.in1, c.expected, c.expErrMsg, c.expCounterResetCollision)
+			testHistogramAdd(t, c.in1, c.in2, c.expected, c.expErrMsg, c.expCounterResetCollision, c.expNHCBBoundsReconciled)
+			testHistogramAdd(t, c.in2, c.in1, c.expected, c.expErrMsg, c.expCounterResetCollision, c.expNHCBBoundsReconciled)
 		})
 	}
 }
 
-func testHistogramAdd(t *testing.T, a, b, expected *FloatHistogram, expErrMsg string, expCounterResetCollision bool) {
+func testHistogramAdd(t *testing.T, a, b, expected *FloatHistogram, expErrMsg string, expCounterResetCollision, expNHCBBoundsReconciled bool) {
+	require.NoError(t, a.Validate(), "a")
+	require.NoError(t, b.Validate(), "b")
+
 	var (
 		aCopy        = a.Copy()
 		bCopy        = b.Copy()
@@ -2324,7 +2532,7 @@ func testHistogramAdd(t *testing.T, a, b, expected *FloatHistogram, expErrMsg st
 		expectedCopy = expected.Copy()
 	}
 
-	res, warn, err := aCopy.Add(bCopy)
+	res, counterResetCollision, nhcbBoundsReconciled, err := aCopy.Add(bCopy)
 	if expErrMsg != "" {
 		require.EqualError(t, err, expErrMsg)
 	} else {
@@ -2332,7 +2540,8 @@ func testHistogramAdd(t *testing.T, a, b, expected *FloatHistogram, expErrMsg st
 	}
 
 	// Check that the warnings are correct.
-	require.Equal(t, expCounterResetCollision, warn)
+	require.Equal(t, expCounterResetCollision, counterResetCollision)
+	require.Equal(t, expNHCBBoundsReconciled, nhcbBoundsReconciled)
 
 	if expected != nil {
 		res.Compact(0)
@@ -2356,6 +2565,7 @@ func TestFloatHistogramSub(t *testing.T) {
 		in1, in2, expected       *FloatHistogram
 		expErrMsg                string
 		expCounterResetCollision bool
+		expNHCBBoundsReconciled  bool
 	}{
 		{
 			name: "same bucket layout",
@@ -2433,34 +2643,7 @@ func TestFloatHistogramSub(t *testing.T) {
 				Sum:             23,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
-			},
-			in2: &FloatHistogram{
-				Schema:          CustomBucketsSchema,
-				Count:           11,
-				Sum:             12,
-				PositiveSpans:   []Span{{0, 2}, {1, 3}},
-				PositiveBuckets: []float64{0, 0, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
-			},
-			expected: &FloatHistogram{
-				Schema:          CustomBucketsSchema,
-				Count:           4,
-				Sum:             11,
-				PositiveSpans:   []Span{{0, 2}, {1, 3}},
-				PositiveBuckets: []float64{1, 0, 1, 1, 1},
-				CustomValues:    []float64{1, 2, 3, 4},
-			},
-		},
-		{
-			name: "different custom bucket layout",
-			in1: &FloatHistogram{
-				Schema:          CustomBucketsSchema,
-				Count:           15,
-				Sum:             23,
-				PositiveSpans:   []Span{{0, 2}, {1, 3}},
-				PositiveBuckets: []float64{1, 0, 3, 4, 7},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
 			in2: &FloatHistogram{
 				Schema:          CustomBucketsSchema,
@@ -2470,7 +2653,45 @@ func TestFloatHistogramSub(t *testing.T) {
 				PositiveBuckets: []float64{0, 0, 2, 3, 6},
 				CustomValues:    []float64{1, 2, 3, 4, 5},
 			},
-			expErrMsg: "cannot apply this operation on custom buckets histograms with different custom bounds",
+			expected: &FloatHistogram{
+				Schema:          CustomBucketsSchema,
+				Count:           4,
+				Sum:             11,
+				PositiveSpans:   []Span{{0, 2}, {1, 3}},
+				PositiveBuckets: []float64{1, 0, 1, 1, 1},
+				CustomValues:    []float64{1, 2, 3, 4, 5},
+			},
+		},
+		{
+			name: "different custom bucket layout - with intersection and rollup",
+			in1: &FloatHistogram{
+				Schema:           CustomBucketsSchema,
+				Count:            220,
+				Sum:              9.9,
+				PositiveSpans:    []Span{{0, 2}, {1, 1}, {0, 2}},
+				PositiveBuckets:  []float64{10, 20, 40, 50, 100},
+				CustomValues:     []float64{1, 2, 3, 4, 5},
+				CounterResetHint: GaugeType,
+			},
+			in2: &FloatHistogram{
+				Schema:           CustomBucketsSchema,
+				Count:            6,
+				Sum:              4.4,
+				PositiveSpans:    []Span{{0, 1}, {1, 1}},
+				PositiveBuckets:  []float64{1, 5},
+				CustomValues:     []float64{2, 4},
+				CounterResetHint: GaugeType,
+			},
+			expected: &FloatHistogram{
+				Schema:           CustomBucketsSchema,
+				Count:            214,
+				Sum:              5.5,
+				PositiveSpans:    []Span{{0, 3}},
+				PositiveBuckets:  []float64{10 + 20 - 1, 40, 50 + 100 - 5},
+				CustomValues:     []float64{2, 4},
+				CounterResetHint: GaugeType,
+			},
+			expNHCBBoundsReconciled: true,
 		},
 		{
 			name: "mix exponential and custom buckets histograms",
@@ -2491,7 +2712,7 @@ func TestFloatHistogramSub(t *testing.T) {
 				Sum:             12,
 				PositiveSpans:   []Span{{0, 2}, {1, 3}},
 				PositiveBuckets: []float64{0, 0, 2, 3, 6},
-				CustomValues:    []float64{1, 2, 3, 4},
+				CustomValues:    []float64{1, 2, 3, 4, 5, 6},
 			},
 			expErrMsg: "cannot apply this operation on histograms with a mix of exponential and custom bucket schemas",
 		},
@@ -2512,7 +2733,7 @@ func TestFloatHistogramSub(t *testing.T) {
 
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			testFloatHistogramSub(t, c.in1, c.in2, c.expected, c.expErrMsg, c.expCounterResetCollision)
+			testFloatHistogramSub(t, c.in1, c.in2, c.expected, c.expErrMsg, c.expCounterResetCollision, c.expNHCBBoundsReconciled)
 
 			var expectedNegative *FloatHistogram
 			if c.expected != nil {
@@ -2522,12 +2743,15 @@ func TestFloatHistogramSub(t *testing.T) {
 				// counter reset hint for this test.
 				expectedNegative.CounterResetHint = c.expected.CounterResetHint
 			}
-			testFloatHistogramSub(t, c.in2, c.in1, expectedNegative, c.expErrMsg, c.expCounterResetCollision)
+			testFloatHistogramSub(t, c.in2, c.in1, expectedNegative, c.expErrMsg, c.expCounterResetCollision, c.expNHCBBoundsReconciled)
 		})
 	}
 }
 
-func testFloatHistogramSub(t *testing.T, a, b, expected *FloatHistogram, expErrMsg string, expCounterResetCollision bool) {
+func testFloatHistogramSub(t *testing.T, a, b, expected *FloatHistogram, expErrMsg string, expCounterResetCollision, expNHCBBoundsReconciled bool) {
+	require.NoError(t, a.Validate(), "a")
+	require.NoError(t, b.Validate(), "b")
+
 	var (
 		aCopy        = a.Copy()
 		bCopy        = b.Copy()
@@ -2538,7 +2762,7 @@ func testFloatHistogramSub(t *testing.T, a, b, expected *FloatHistogram, expErrM
 		expectedCopy = expected.Copy()
 	}
 
-	res, warn, err := aCopy.Sub(bCopy)
+	res, counterResetCollision, nhcbBoundsReconciled, err := aCopy.Sub(bCopy)
 	if expErrMsg != "" {
 		require.EqualError(t, err, expErrMsg)
 	} else {
@@ -2558,7 +2782,8 @@ func testFloatHistogramSub(t *testing.T, a, b, expected *FloatHistogram, expErrM
 		require.Equal(t, b, bCopy)
 
 		// Check that the warnings are correct.
-		require.Equal(t, expCounterResetCollision, warn)
+		require.Equal(t, expCounterResetCollision, counterResetCollision)
+		require.Equal(t, expNHCBBoundsReconciled, nhcbBoundsReconciled)
 	}
 }
 
@@ -3440,8 +3665,9 @@ func TestFloatHistogramSize(t *testing.T) {
 				},
 				PositiveBuckets: []float64{1, 3.3, 4.2, 0.1}, // 24 bytes + 4 * 8 bytes.
 				NegativeSpans: []Span{ // 24 bytes.
-					{3, 2},  // 2 * 4 bytes.
-					{3, 2}}, //  2 * 4 bytes.
+					{3, 2}, // 2 * 4 bytes.
+					{3, 2}, //  2 * 4 bytes.
+				},
 				NegativeBuckets: []float64{3.1, 3, 1.234e5, 1000}, // 24 bytes + 4 * 8 bytes.
 				CustomValues:    nil,                              // 24 bytes.
 			},

model/histogram/generic.go

@@ -40,7 +40,6 @@ var (
 	ErrHistogramCustomBucketsInfinite   = errors.New("histogram custom bounds must be finite")
 	ErrHistogramCustomBucketsNaN        = errors.New("histogram custom bounds must not be NaN")
 	ErrHistogramsIncompatibleSchema     = errors.New("cannot apply this operation on histograms with a mix of exponential and custom bucket schemas")
-	ErrHistogramsIncompatibleBounds     = errors.New("cannot apply this operation on custom buckets histograms with different custom bounds")
 	ErrHistogramCustomBucketsZeroCount  = errors.New("custom buckets: must have zero count of 0")
 	ErrHistogramCustomBucketsZeroThresh = errors.New("custom buckets: must have zero threshold of 0")
 	ErrHistogramCustomBucketsNegSpans   = errors.New("custom buckets: must not have negative spans")