check if the acquis tomb is dying while processing logs in replay mode for file/s3/docker (#2152)
This commit is contained in:
parent
3132aa54b7
commit
0279e549bd
|
@ -311,21 +311,26 @@ func (d *DockerSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) er
|
||||||
scanner = bufio.NewScanner(reader)
|
scanner = bufio.NewScanner(reader)
|
||||||
}
|
}
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
line := scanner.Text()
|
select {
|
||||||
if line == "" {
|
case <-t.Dying():
|
||||||
continue
|
d.logger.Infof("Shutting down reader for container %s", containerConfig.Name)
|
||||||
|
default:
|
||||||
|
line := scanner.Text()
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
l := types.Line{}
|
||||||
|
l.Raw = line
|
||||||
|
l.Labels = d.Config.Labels
|
||||||
|
l.Time = time.Now().UTC()
|
||||||
|
l.Src = containerConfig.Name
|
||||||
|
l.Process = true
|
||||||
|
l.Module = d.GetName()
|
||||||
|
linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc()
|
||||||
|
evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
||||||
|
out <- evt
|
||||||
|
d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw)
|
||||||
}
|
}
|
||||||
l := types.Line{}
|
|
||||||
l.Raw = line
|
|
||||||
l.Labels = d.Config.Labels
|
|
||||||
l.Time = time.Now().UTC()
|
|
||||||
l.Src = containerConfig.Name
|
|
||||||
l.Process = true
|
|
||||||
l.Module = d.GetName()
|
|
||||||
linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc()
|
|
||||||
evt := types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
|
||||||
out <- evt
|
|
||||||
d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw)
|
|
||||||
}
|
}
|
||||||
err = scanner.Err()
|
err = scanner.Err()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
@ -514,22 +514,28 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom
|
||||||
scanner.Buffer(buf, f.config.MaxBufferSize)
|
scanner.Buffer(buf, f.config.MaxBufferSize)
|
||||||
}
|
}
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
if scanner.Text() == "" {
|
select {
|
||||||
continue
|
case <-t.Dying():
|
||||||
}
|
logger.Infof("File datasource %s stopping", filename)
|
||||||
l := types.Line{
|
return nil
|
||||||
Raw: scanner.Text(),
|
default:
|
||||||
Time: time.Now().UTC(),
|
if scanner.Text() == "" {
|
||||||
Src: filename,
|
continue
|
||||||
Labels: f.config.Labels,
|
}
|
||||||
Process: true,
|
l := types.Line{
|
||||||
Module: f.GetName(),
|
Raw: scanner.Text(),
|
||||||
}
|
Time: time.Now().UTC(),
|
||||||
logger.Debugf("line %s", l.Raw)
|
Src: filename,
|
||||||
linesRead.With(prometheus.Labels{"source": filename}).Inc()
|
Labels: f.config.Labels,
|
||||||
|
Process: true,
|
||||||
|
Module: f.GetName(),
|
||||||
|
}
|
||||||
|
logger.Debugf("line %s", l.Raw)
|
||||||
|
linesRead.With(prometheus.Labels{"source": filename}).Inc()
|
||||||
|
|
||||||
//we're reading logs at once, it must be time-machine buckets
|
//we're reading logs at once, it must be time-machine buckets
|
||||||
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if err := scanner.Err(); err != nil {
|
if err := scanner.Err(); err != nil {
|
||||||
logger.Errorf("Error while reading file: %s", err)
|
logger.Errorf("Error while reading file: %s", err)
|
||||||
|
|
|
@ -418,23 +418,29 @@ func (s *S3Source) readFile(bucket string, key string) error {
|
||||||
scanner.Buffer(buf, s.Config.MaxBufferSize)
|
scanner.Buffer(buf, s.Config.MaxBufferSize)
|
||||||
}
|
}
|
||||||
for scanner.Scan() {
|
for scanner.Scan() {
|
||||||
text := scanner.Text()
|
select {
|
||||||
logger.Tracef("Read line %s", text)
|
case <-s.t.Dying():
|
||||||
linesRead.WithLabelValues(bucket).Inc()
|
s.logger.Infof("Shutting down reader for %s/%s", bucket, key)
|
||||||
l := types.Line{}
|
return nil
|
||||||
l.Raw = text
|
default:
|
||||||
l.Labels = s.Config.Labels
|
text := scanner.Text()
|
||||||
l.Time = time.Now().UTC()
|
logger.Tracef("Read line %s", text)
|
||||||
l.Process = true
|
linesRead.WithLabelValues(bucket).Inc()
|
||||||
l.Module = s.GetName()
|
l := types.Line{}
|
||||||
l.Src = bucket
|
l.Raw = text
|
||||||
var evt types.Event
|
l.Labels = s.Config.Labels
|
||||||
if !s.Config.UseTimeMachine {
|
l.Time = time.Now().UTC()
|
||||||
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE}
|
l.Process = true
|
||||||
} else {
|
l.Module = s.GetName()
|
||||||
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
l.Src = bucket
|
||||||
|
var evt types.Event
|
||||||
|
if !s.Config.UseTimeMachine {
|
||||||
|
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.LIVE}
|
||||||
|
} else {
|
||||||
|
evt = types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE}
|
||||||
|
}
|
||||||
|
s.out <- evt
|
||||||
}
|
}
|
||||||
s.out <- evt
|
|
||||||
}
|
}
|
||||||
if err := scanner.Err(); err != nil {
|
if err := scanner.Err(); err != nil {
|
||||||
return fmt.Errorf("failed to read object %s/%s: %s", bucket, key, err)
|
return fmt.Errorf("failed to read object %s/%s: %s", bucket, key, err)
|
||||||
|
@ -629,6 +635,7 @@ func (s *S3Source) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error
|
||||||
s.out = out
|
s.out = out
|
||||||
s.ctx, s.cancel = context.WithCancel(context.Background())
|
s.ctx, s.cancel = context.WithCancel(context.Background())
|
||||||
s.Config.UseTimeMachine = true
|
s.Config.UseTimeMachine = true
|
||||||
|
s.t = t
|
||||||
if s.Config.Key != "" {
|
if s.Config.Key != "" {
|
||||||
err := s.readFile(s.Config.BucketName, s.Config.Key)
|
err := s.readFile(s.Config.BucketName, s.Config.Key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
Loading…
Reference in a new issue