问题描述：
在go程序中，通过执行一个命令启动一个子命令，并通过pipe读取子程序的标准输入和输出，通过scanner默认按行读取，此时如果子程序输出时没有携带’\n’，scanner就不会打印输出，而是会累积到缓存buf上限，最终被丢弃，直到遇到一个\n，然后输出所有的内容，默认buf缓存上限时65536，如果日志打印处还有限制，如glog就限制最大的打印字节数为4096，那么就会导致日志再次丢失。
解决方法：
不适用scanner去按行读取，直接读取管道的内容，然后设置上限，超过时或者遇到’\n’时打印
测试代码：
子程序：
#include <stdio.h>
#include <unistd.h>int main() {
int count = 0;while (1) {fprintf(stderr, "%d", count);count = (count + 1) % 10;usleep(500); // Sleep for 500,000 microseconds (0.5 seconds)
}return 0;
}
主程序：
package mainimport ("bufio""fmt""os/exec""strings""log"
)func main() {cmd := exec.Command("./test")stdout, err := cmd.StdoutPipe()if err != nil {fmt.Println("Error creating StdoutPipe:", err)return}cmd.Stderr = cmd.Stdouterr = cmd.Start()if err != nil {fmt.Println("Error starting command:", err)return}scanner := bufio.NewScanner(stdout)// scanner.Split(bufio.ScanBytes)// buf := ""// for scanner.Scan() {// buf += scanner.Text()// if strings.Contains(buf, "\n") || len(buf) >= 256 {// log.Printf("%s", buf)// buf = ""// }// }for scanner.Scan() {log.Printf("%s", scanner.Text())}if err := scanner.Err(); err != nil {fmt.Println("Error reading standard output:", err)}err = cmd.Wait()if err != nil {fmt.Println("Error waiting for command to finish:", err)}
}修改程序：
package mainimport ("bufio""fmt""io""log""os/exec"
)func getReaderSize(rd io.Reader) {b, ok := rd.(*bufio.Reader)if ok {log.Printf("rd size: %d", b.Size())} else {log.Printf("rd is not bufio.Reader")}
}func main() {// Command to executecmd := exec.Command("./test")// Create a pipe to capture the standard output of the commandstdout, err := cmd.StdoutPipe()if err != nil {fmt.Println("Error creating StdoutPipe:", err)return}cmd.Stderr = cmd.Stdout// Start the commanderr = cmd.Start()if err != nil {fmt.Println("Error starting command:", err)return} Create a scanner to read the command's standard output//scanner := bufio.NewScanner(stdout)//scanner.Split(bufio.ScanBytes)// Read and print each line from the output//buf := make([]byte, 256)//bufLen := 0//for scanner.Scan() {// buf[bufLen] = scanner.Bytes()[0]// // buf = append(buf, scanner.Bytes()...)// bufLen += 1// if buf[bufLen-1] == '\n' || bufLen >= 256 {// log.Printf("%s", string(buf[:bufLen]))// bufLen = 0// }//}// Check for errors in scanning//if err := scanner.Err(); err != nil {// fmt.Println("Error reading standard output:", err)//}// Create a buffered reader to read from the command's stdoutreader := bufio.NewReaderSize(stdout, 256)getReaderSize(stdout)log.Printf("reader size: %d", reader.Size()) Buffer to store incomplete lines//var incompleteLine []byte// Buffer to read chunks of bytes//chunk := make([]byte, 256)////for {// // Read a chunk of bytes// n, err := reader.Read(chunk)// if err != nil {// break // Break the loop when an error occurs (e.g., when the command finishes)// }//// // Process each byte in the chunk// for i := 0; i < n; i++ {// b := chunk[i]//// // Check for newline or length exceeding 256// if b == '\n' || len(incompleteLine) >= 256 {// // Print the line// log.Printf("%s", incompleteLine)//// // Reset the incomplete line buffer// incompleteLine = nil// } else {// // Add the byte to the incomplete line buffer// incompleteLine = append(incompleteLine, b)// }// }//}for {s, err := reader.ReadSlice('\n')if err != nil && err != bufio.ErrBufferFull {if len(s) > 0 {log.Printf("reader err but exist data, reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))}fmt.Println("Error reader ReadString:", err)break // Break the loop when an error occurs (e.g., when the command finishes)}log.Printf("reader size: %d, read string size: %d, string: %s", reader.Size(), len(s), string(s))}// Wait for the command to finisherr = cmd.Wait()if err != nil {fmt.Println("Error waiting for command to finish:", err)}
}benchmark test:
package mainimport ("strconv""strings""testing"
)func stringTest1() string {var buf stringfor i := 0; i < 256; i++ {buf += strconv.Itoa(i)}return buf
}func stringTest2() string {var buf strings.Builderfor i := 0; i < 256; i++ {buf.Write([]byte(strconv.Itoa(i)))}return buf.String()
}func stringTest3() string {var buf = make([]byte, 0)for i := 0; i < 256; i++ {buf = append(buf, []byte(strconv.Itoa(i))...)}return string(buf)
}func stringTest4() string {var buf = make([]byte, 256)for i := 0; i < 256; i++ {buf[i] = '1'}return string(buf)
}func BenchmarkStringTest1(b *testing.B) {for i := 0; i < b.N; i++ {stringTest1()}
}
func BenchmarkStringTest2(b *testing.B) {for i := 0; i < b.N; i++ {stringTest2()}
}
func BenchmarkStringTest3(b *testing.B) {for i := 0; i < b.N; i++ {stringTest3()}
}
func BenchmarkStringTest4(b *testing.B) {for i := 0; i < b.N; i++ {stringTest4()}
}
benchmark test
cmd:
go test -bench . -benchmem
go test -bench=<function>
)
