Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions app/cli/internal/trace/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"io"
"sort"
"strconv"
"time"

"github.com/chainloop-dev/chainloop/app/cli/internal/trace/pointer"
"github.com/chainloop-dev/chainloop/app/cli/internal/trace/skill"
Expand Down Expand Up @@ -237,7 +238,8 @@ type SkillUse struct {
// skill has no folder (a skill built into the agent) or the session data
// does not tell. A use with no folder is not recorded.
Dir string
// FirstUsedAt is the time of the first use, RFC3339.
// FirstUsedAt is the time of the first use, RFC3339 in UTC with whole
// seconds, or "" when no use has a time that can be read.
FirstUsedAt string
// ByModel counts the uses that the model started with a tool call.
ByModel int
Expand All @@ -248,7 +250,8 @@ type SkillUse struct {
}

// Record adds one use at time at, RFC3339, to the counts. The first use is
// the earliest time recorded, whatever the order of the calls.
// the earliest time recorded, whatever the order of the calls. A use whose
// time cannot be read is counted, but its time is ignored.
func (u *SkillUse) Record(at string, byModel, subagent bool) {
u.noteTime(at)
if byModel {
Expand All @@ -272,24 +275,41 @@ func (u *SkillUse) Merge(other SkillUse) {
}
}

// noteTime keeps the earliest time of use. It stores the time as RFC3339 in
// UTC with whole seconds, so that FirstUsedAt has one format for all
// providers, and so that a string comparison gives the time order. A time
// that cannot be read is ignored.
func (u *SkillUse) noteTime(at string) {
if at != "" && (u.FirstUsedAt == "" || at < u.FirstUsedAt) {
t, err := time.Parse(time.RFC3339Nano, at)
if err != nil {
return
}

at = t.UTC().Format(time.RFC3339)
if u.FirstUsedAt == "" || at < u.FirstUsedAt {
u.FirstUsedAt = at
}
}

// SortedSkillUses returns the uses in the order of their first use. The name
// breaks ties, so that the same session data always gives the same order:
// the limit on skill entries keeps the first ones.
// SortedSkillUses returns the uses in the order of their first use. The uses
// with no time come last, so that the limit on skill entries, which keeps the
// first ones, does not drop a skill with a known time for one with no time.
// The name breaks ties, so that the same session data always gives the same
// order.
func SortedSkillUses(uses map[string]*SkillUse) []SkillUse {
out := make([]SkillUse, 0, len(uses))
for _, use := range uses {
out = append(out, *use)
}

sort.Slice(out, func(i, j int) bool {
if out[i].FirstUsedAt != out[j].FirstUsedAt {
return out[i].FirstUsedAt < out[j].FirstUsedAt
a, b := out[i].FirstUsedAt, out[j].FirstUsedAt
if a != b {
if a == "" || b == "" {
return b == ""
}

return a < b
}

return out[i].Name < out[j].Name
Expand Down
141 changes: 141 additions & 0 deletions app/cli/internal/trace/provider_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
//
// Copyright 2026 The Chainloop Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package trace

import (
"testing"

"github.com/stretchr/testify/assert"
)

const (
tenOClock = "2026-10-07T10:00:00Z"
tenOClockPlusTwo = "2026-10-07T12:00:00+02:00"
)

func TestSkillUseRecordTime(t *testing.T) {
cases := []struct {
name string
at []string
want string
}{
{name: "RFC3339 in UTC is kept", at: []string{tenOClock}, want: tenOClock},
{name: "fractional seconds are dropped", at: []string{"2026-10-07T10:00:00.123Z"}, want: tenOClock},
{name: "an offset is converted to UTC", at: []string{tenOClockPlusTwo}, want: tenOClock},
{name: "an empty time is ignored", at: []string{""}, want: ""},
{name: "an unreadable time is ignored", at: []string{"yesterday"}, want: ""},
{name: "a readable time after an unreadable one is kept", at: []string{"yesterday", tenOClock}, want: tenOClock},
{name: "an unreadable time does not replace a readable one", at: []string{tenOClock, "0"}, want: tenOClock},
{
// As raw strings, "2026-10-07T10:30:00Z" sorts before
// tenOClockPlusTwo, which is 10:00 in UTC.
name: "the earliest time wins across offsets",
at: []string{tenOClockPlusTwo, "2026-10-07T10:30:00Z"},
want: tenOClock,
},
{name: "the earliest time wins in any order", at: []string{"2026-10-07T10:30:00.500Z", "2026-10-07T10:00:00.900Z"}, want: tenOClock},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var use SkillUse
for _, at := range tc.at {
use.Record(at, true, false)
}

assert.Equal(t, tc.want, use.FirstUsedAt)
assert.Equal(t, len(tc.at), use.ByModel, "each use counts, also with no readable time")
})
}
}

func TestSkillUseMergeTime(t *testing.T) {
cases := []struct {
name string
first string
other string
want string
}{
{name: "an earlier time of the other use wins", first: "2026-10-07T10:30:00Z", other: "2026-10-07T10:00:00.250Z", want: tenOClock},
{name: "a later time of the other use is ignored", first: tenOClock, other: "2026-10-07T13:00:00+02:00", want: tenOClock},
{name: "the other time fills an unknown time", first: "", other: tenOClockPlusTwo, want: tenOClock},
{name: "an unreadable other time is ignored", first: tenOClock, other: "soon", want: tenOClock},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
use := SkillUse{Name: "review"}
if tc.first != "" {
use.Record(tc.first, true, false)
}

use.Merge(SkillUse{Name: "review", FirstUsedAt: tc.other, ByUser: 1})

assert.Equal(t, tc.want, use.FirstUsedAt)
})
}
}

func TestSortedSkillUses(t *testing.T) {
cases := []struct {
name string
uses []SkillUse
want []string
}{
{
name: "the order of first use",
uses: []SkillUse{
{Name: "b", FirstUsedAt: "2026-10-07T10:05:00Z"},
{Name: "a", FirstUsedAt: "2026-10-07T10:10:00Z"},
},
want: []string{"b", "a"},
},
{
name: "the name breaks ties",
uses: []SkillUse{
{Name: "b", FirstUsedAt: tenOClock},
{Name: "a", FirstUsedAt: tenOClock},
},
want: []string{"a", "b"},
},
{
name: "skills with no time come last, by name",
uses: []SkillUse{
{Name: "c"},
{Name: "z", FirstUsedAt: "2026-10-07T10:10:00Z"},
{Name: "a"},
{Name: "y", FirstUsedAt: tenOClock},
},
want: []string{"y", "z", "a", "c"},
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
uses := make(map[string]*SkillUse, len(tc.uses))
for i := range tc.uses {
uses[tc.uses[i].Name] = &tc.uses[i]
}

var got []string
for _, use := range SortedSkillUses(uses) {
got = append(got, use.Name)
}

assert.Equal(t, tc.want, got)
})
}
}
4 changes: 4 additions & 0 deletions app/cli/pkg/action/trace_skills.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,10 @@ func attachSkills(ctx context.Context, adder specMaterialAdder, redactor *specRe
if warning != "" {
warnings = append(warnings, warning)
}
if s.use.FirstUsedAt == "" {
// The entry got the push time, which is later than the use.
warnings = append(warnings, fmt.Sprintf("the time of the first use of skill %q is unknown; the push time is recorded", s.use.Name))
}

entries = append(entries, entry)
}
Expand Down
32 changes: 32 additions & 0 deletions app/cli/pkg/action/trace_skills_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,38 @@ func TestAttachSkills(t *testing.T) {
assert.Equal(t, []string{`skill "asd-ste100" was not recorded`}, warnings)
})

t.Run("an entry with no time of use gets the push time, with a warning", func(t *testing.T) {
const unknownTime = `the time of the first use of skill "asd-ste100" is unknown; the push time is recorded`
cases := []struct {
name string
at string
failOn map[string]bool
entries int
warnings []string
}{
{name: "known time", at: skillTenOClock, entries: 1},
{name: "unknown time", entries: 1, warnings: []string{unknownTime}},
{
name: "no time warning for a skill that is not recorded",
failOn: map[string]bool{"spec-7412a0-skill-asd-ste100": true},
warnings: []string{`skill "asd-ste100" was not recorded`},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: steSkillName, FirstUsedAt: tc.at, ByModel: 1})

adder := &fakeMaterialAdder{failOn: tc.failOn}
entries, warnings := attachSkills(context.Background(), adder, newSpecRedactor(t.TempDir(), false), materials.NewNameAllocator(nil), skillSessionID, []sessionSkill{s}, zerolog.Nop())
require.Len(t, entries, tc.entries)
assert.Equal(t, tc.warnings, warnings)
for _, entry := range entries {
assert.NotEmpty(t, entry.CapturedAt)
}
})
}
})

t.Run("a plugin skill gets a valid material name", func(t *testing.T) {
s := stored(t, map[string]string{skill.DefinitionFile: skillDoc}, trace.SkillUse{Name: "superpowers:brainstorming", FirstUsedAt: skillTenOClock, ByModel: 1})

Expand Down
Loading