QUIC sniffer: handle multiple initial packets (#3802)

* QUIC sniffer: handle multiple initial packets
Basically copied from Vigilans/v2ray-core@8f33db0

Co-Authored-By: Vigilans <vigilans@foxmail.com>

* Remove unnecessary file

---------

Co-authored-by: Vigilans <vigilans@foxmail.com>

app/dispatcher/default.go

@@ -41,8 +41,14 @@ func (r *cachedReader) Cache(b *buf.Buffer) {
 	if !mb.IsEmpty() {
 		r.cache, _ = buf.MergeMulti(r.cache, mb)
 	}
-	b.Clear()
+	cacheLen := r.cache.Len()
-	rawBytes := b.Extend(buf.Size)
+	if cacheLen <= b.Cap() {
+		b.Clear()
+	} else {
+		b.Release()
+		*b = *buf.NewWithSize(cacheLen)
+	}
+	rawBytes := b.Extend(cacheLen)
 	n := r.cache.Copy(rawBytes)
 	b.Resize(0, int32(n))
 	r.Unlock()

common/buf/buffer.go

@@ -207,6 +207,21 @@ func (b *Buffer) Len() int32 {
 	return b.end - b.start
 }
 
+// Cap returns the capacity of the buffer content.
+func (b *Buffer) Cap() int32 {
+	if b == nil {
+		return 0
+	}
+	return int32(len(b.v))
+}
+
+// NewWithSize creates a Buffer with 0 length and capacity with at least the given size.
+func NewWithSize(size int32) *Buffer {
+	return &Buffer{
+		v:         bytespool.Alloc(size),
+	}
+}
+
 // IsEmpty returns true if the buffer is empty.
 func (b *Buffer) IsEmpty() bool {
 	return b.Len() == 0

common/protocol/quic/sniff.go

@@ -47,206 +47,224 @@ var (
 )
 
 func SniffQUIC(b []byte) (*SniffHeader, error) {
-	buffer := buf.FromBytes(b)
+	// Crypto data separated across packets
-	typeByte, err := buffer.ReadByte()
+	cryptoLen := 0
-	if err != nil {
+	cryptoData := bytespool.Alloc(int32(len(b)))
-		return nil, errNotQuic
+	defer bytespool.Free(cryptoData)
-	}
-	isLongHeader := typeByte&0x80 > 0
-	if !isLongHeader || typeByte&0x40 == 0 {
-		return nil, errNotQuicInitial
-	}
 
-	vb, err := buffer.ReadBytes(4)
+	// Parse QUIC packets
-	if err != nil {
+	for len(b) > 0 {
-		return nil, errNotQuic
+		buffer := buf.FromBytes(b)
-	}
+		typeByte, err := buffer.ReadByte()
+		if err != nil {
+			return nil, errNotQuic
+		}
 
-	versionNumber := binary.BigEndian.Uint32(vb)
+		isLongHeader := typeByte&0x80 > 0
+		if !isLongHeader || typeByte&0x40 == 0 {
+			return nil, errNotQuicInitial
+		}
 
-	if versionNumber != 0 && typeByte&0x40 == 0 {
+		vb, err := buffer.ReadBytes(4)
-		return nil, errNotQuic
+		if err != nil {
-	} else if versionNumber != versionDraft29 && versionNumber != version1 {
+			return nil, errNotQuic
-		return nil, errNotQuic
+		}
-	}
 
-	if (typeByte&0x30)>>4 != 0x0 {
+		versionNumber := binary.BigEndian.Uint32(vb)
-		return nil, errNotQuicInitial
+		if versionNumber != 0 && typeByte&0x40 == 0 {
-	}
+			return nil, errNotQuic
+		} else if versionNumber != versionDraft29 && versionNumber != version1 {
+			return nil, errNotQuic
+		}
 
-	var destConnID []byte
+		packetType := (typeByte & 0x30) >> 4
-	if l, err := buffer.ReadByte(); err != nil {
+		isQuicInitial := packetType == 0x0
-		return nil, errNotQuic
-	} else if destConnID, err = buffer.ReadBytes(int32(l)); err != nil {
-		return nil, errNotQuic
-	}
 
-	if l, err := buffer.ReadByte(); err != nil {
+		var destConnID []byte
-		return nil, errNotQuic
+		if l, err := buffer.ReadByte(); err != nil {
-	} else if common.Error2(buffer.ReadBytes(int32(l))) != nil {
+			return nil, errNotQuic
-		return nil, errNotQuic
+		} else if destConnID, err = buffer.ReadBytes(int32(l)); err != nil {
-	}
+			return nil, errNotQuic
+		}
 
-	tokenLen, err := quicvarint.Read(buffer)
+		if l, err := buffer.ReadByte(); err != nil {
-	if err != nil || tokenLen > uint64(len(b)) {
+			return nil, errNotQuic
-		return nil, errNotQuic
+		} else if common.Error2(buffer.ReadBytes(int32(l))) != nil {
-	}
+			return nil, errNotQuic
+		}
 
-	if _, err = buffer.ReadBytes(int32(tokenLen)); err != nil {
+		tokenLen, err := quicvarint.Read(buffer)
-		return nil, errNotQuic
+		if err != nil || tokenLen > uint64(len(b)) {
-	}
+			return nil, errNotQuic
+		}
 
-	packetLen, err := quicvarint.Read(buffer)
+		if _, err = buffer.ReadBytes(int32(tokenLen)); err != nil {
-	if err != nil {
+			return nil, errNotQuic
-		return nil, errNotQuic
+		}
-	}
 
-	hdrLen := len(b) - int(buffer.Len())
+		packetLen, err := quicvarint.Read(buffer)
+		if err != nil {
+			return nil, errNotQuic
+		}
 
-	origPNBytes := make([]byte, 4)
+		hdrLen := len(b) - int(buffer.Len())
-	copy(origPNBytes, b[hdrLen:hdrLen+4])
+		if len(b) < hdrLen+int(packetLen) {
+			return nil, common.ErrNoClue // Not enough data to read as a QUIC packet. QUIC is UDP-based, so this is unlikely to happen.
+		}
 
-	var salt []byte
+		restPayload := b[hdrLen+int(packetLen):]
-	if versionNumber == version1 {
+		if !isQuicInitial { // Skip this packet if it's not initial packet
-		salt = quicSalt
+			b = restPayload
-	} else {
+			continue
-		salt = quicSaltOld
+		}
-	}
-	initialSecret := hkdf.Extract(crypto.SHA256.New, destConnID, salt)
-	secret := hkdfExpandLabel(crypto.SHA256, initialSecret, []byte{}, "client in", crypto.SHA256.Size())
-	hpKey := hkdfExpandLabel(initialSuite.Hash, secret, []byte{}, "quic hp", initialSuite.KeyLen)
-	block, err := aes.NewCipher(hpKey)
-	if err != nil {
-		return nil, err
-	}
 
-	cache := buf.New()
+		origPNBytes := make([]byte, 4)
-	defer cache.Release()
+		copy(origPNBytes, b[hdrLen:hdrLen+4])
 
-	mask := cache.Extend(int32(block.BlockSize()))
+		var salt []byte
-	block.Encrypt(mask, b[hdrLen+4:hdrLen+4+16])
+		if versionNumber == version1 {
-	b[0] ^= mask[0] & 0xf
+			salt = quicSalt
-	for i := range b[hdrLen : hdrLen+4] {
+		} else {
-		b[hdrLen+i] ^= mask[i+1]
+			salt = quicSaltOld
-	}
+		}
-	packetNumberLength := b[0]&0x3 + 1
+		initialSecret := hkdf.Extract(crypto.SHA256.New, destConnID, salt)
-	if packetNumberLength != 1 {
+		secret := hkdfExpandLabel(crypto.SHA256, initialSecret, []byte{}, "client in", crypto.SHA256.Size())
-		return nil, errNotQuicInitial
+		hpKey := hkdfExpandLabel(initialSuite.Hash, secret, []byte{}, "quic hp", initialSuite.KeyLen)
-	}
+		block, err := aes.NewCipher(hpKey)
-	var packetNumber uint32
-	{
-		n, err := buffer.ReadByte()
 		if err != nil {
 			return nil, err
 		}
-		packetNumber = uint32(n)
-	}
 
-	if packetNumber != 0 && packetNumber != 1 {
+		cache := buf.New()
-		return nil, errNotQuicInitial
+		defer cache.Release()
-	}
 
-	extHdrLen := hdrLen + int(packetNumberLength)
+		mask := cache.Extend(int32(block.BlockSize()))
-	copy(b[extHdrLen:hdrLen+4], origPNBytes[packetNumberLength:])
+		block.Encrypt(mask, b[hdrLen+4:hdrLen+4+16])
-	data := b[extHdrLen : int(packetLen)+hdrLen]
+		b[0] ^= mask[0] & 0xf
-
+		for i := range b[hdrLen : hdrLen+4] {
-	key := hkdfExpandLabel(crypto.SHA256, secret, []byte{}, "quic key", 16)
+			b[hdrLen+i] ^= mask[i+1]
-	iv := hkdfExpandLabel(crypto.SHA256, secret, []byte{}, "quic iv", 12)
-	cipher := AEADAESGCMTLS13(key, iv)
-	nonce := cache.Extend(int32(cipher.NonceSize()))
-	binary.BigEndian.PutUint64(nonce[len(nonce)-8:], uint64(packetNumber))
-	decrypted, err := cipher.Open(b[extHdrLen:extHdrLen], nonce, data, b[:extHdrLen])
-	if err != nil {
-		return nil, err
-	}
-	buffer = buf.FromBytes(decrypted)
-
-	cryptoLen := uint(0)
-	cryptoData := bytespool.Alloc(buffer.Len())
-	defer bytespool.Free(cryptoData)
-	for i := 0; !buffer.IsEmpty(); i++ {
-		frameType := byte(0x0) // Default to PADDING frame
-		for frameType == 0x0 && !buffer.IsEmpty() {
-			frameType, _ = buffer.ReadByte()
 		}
-		switch frameType {
+		packetNumberLength := b[0]&0x3 + 1
-		case 0x00: // PADDING frame
+		if packetNumberLength != 1 {
-		case 0x01: // PING frame
-		case 0x02, 0x03: // ACK frame
-			if _, err = quicvarint.Read(buffer); err != nil { // Field: Largest Acknowledged
-				return nil, io.ErrUnexpectedEOF
-			}
-			if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Delay
-				return nil, io.ErrUnexpectedEOF
-			}
-			ackRangeCount, err := quicvarint.Read(buffer) // Field: ACK Range Count
-			if err != nil {
-				return nil, io.ErrUnexpectedEOF
-			}
-			if _, err = quicvarint.Read(buffer); err != nil { // Field: First ACK Range
-				return nil, io.ErrUnexpectedEOF
-			}
-			for i := 0; i < int(ackRangeCount); i++ { // Field: ACK Range
-				if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Range -> Gap
-					return nil, io.ErrUnexpectedEOF
-				}
-				if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Range -> ACK Range Length
-					return nil, io.ErrUnexpectedEOF
-				}
-			}
-			if frameType == 0x03 {
-				if _, err = quicvarint.Read(buffer); err != nil { // Field: ECN Counts -> ECT0 Count
-					return nil, io.ErrUnexpectedEOF
-				}
-				if _, err = quicvarint.Read(buffer); err != nil { // Field: ECN Counts -> ECT1 Count
-					return nil, io.ErrUnexpectedEOF
-				}
-				if _, err = quicvarint.Read(buffer); err != nil { //nolint:misspell // Field: ECN Counts -> ECT-CE Count
-					return nil, io.ErrUnexpectedEOF
-				}
-			}
-		case 0x06: // CRYPTO frame, we will use this frame
-			offset, err := quicvarint.Read(buffer) // Field: Offset
-			if err != nil {
-				return nil, io.ErrUnexpectedEOF
-			}
-			length, err := quicvarint.Read(buffer) // Field: Length
-			if err != nil || length > uint64(buffer.Len()) {
-				return nil, io.ErrUnexpectedEOF
-			}
-			if cryptoLen < uint(offset+length) {
-				cryptoLen = uint(offset + length)
-			}
-			if _, err := buffer.Read(cryptoData[offset : offset+length]); err != nil { // Field: Crypto Data
-				return nil, io.ErrUnexpectedEOF
-			}
-		case 0x1c: // CONNECTION_CLOSE frame, only 0x1c is permitted in initial packet
-			if _, err = quicvarint.Read(buffer); err != nil { // Field: Error Code
-				return nil, io.ErrUnexpectedEOF
-			}
-			if _, err = quicvarint.Read(buffer); err != nil { // Field: Frame Type
-				return nil, io.ErrUnexpectedEOF
-			}
-			length, err := quicvarint.Read(buffer) // Field: Reason Phrase Length
-			if err != nil {
-				return nil, io.ErrUnexpectedEOF
-			}
-			if _, err := buffer.ReadBytes(int32(length)); err != nil { // Field: Reason Phrase
-				return nil, io.ErrUnexpectedEOF
-			}
-		default:
-			// Only above frame types are permitted in initial packet.
-			// See https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2.2-8
 			return nil, errNotQuicInitial
 		}
-	}
+		var packetNumber uint32
+		{
+			n, err := buffer.ReadByte()
+			if err != nil {
+				return nil, err
+			}
+			packetNumber = uint32(n)
+		}
 
-	tlsHdr := &ptls.SniffHeader{}
+		extHdrLen := hdrLen + int(packetNumberLength)
-	err = ptls.ReadClientHello(cryptoData[:cryptoLen], tlsHdr)
+		copy(b[extHdrLen:hdrLen+4], origPNBytes[packetNumberLength:])
-	if err != nil {
+		data := b[extHdrLen : int(packetLen)+hdrLen]
-		return nil, err
+
+		key := hkdfExpandLabel(crypto.SHA256, secret, []byte{}, "quic key", 16)
+		iv := hkdfExpandLabel(crypto.SHA256, secret, []byte{}, "quic iv", 12)
+		cipher := AEADAESGCMTLS13(key, iv)
+		nonce := cache.Extend(int32(cipher.NonceSize()))
+		binary.BigEndian.PutUint64(nonce[len(nonce)-8:], uint64(packetNumber))
+		decrypted, err := cipher.Open(b[extHdrLen:extHdrLen], nonce, data, b[:extHdrLen])
+		if err != nil {
+			return nil, err
+		}
+		buffer = buf.FromBytes(decrypted)
+		for i := 0; !buffer.IsEmpty(); i++ {
+			frameType := byte(0x0) // Default to PADDING frame
+			for frameType == 0x0 && !buffer.IsEmpty() {
+				frameType, _ = buffer.ReadByte()
+			}
+			switch frameType {
+			case 0x00: // PADDING frame
+			case 0x01: // PING frame
+			case 0x02, 0x03: // ACK frame
+				if _, err = quicvarint.Read(buffer); err != nil { // Field: Largest Acknowledged
+					return nil, io.ErrUnexpectedEOF
+				}
+				if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Delay
+					return nil, io.ErrUnexpectedEOF
+				}
+				ackRangeCount, err := quicvarint.Read(buffer) // Field: ACK Range Count
+				if err != nil {
+					return nil, io.ErrUnexpectedEOF
+				}
+				if _, err = quicvarint.Read(buffer); err != nil { // Field: First ACK Range
+					return nil, io.ErrUnexpectedEOF
+				}
+				for i := 0; i < int(ackRangeCount); i++ { // Field: ACK Range
+					if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Range -> Gap
+						return nil, io.ErrUnexpectedEOF
+					}
+					if _, err = quicvarint.Read(buffer); err != nil { // Field: ACK Range -> ACK Range Length
+						return nil, io.ErrUnexpectedEOF
+					}
+				}
+				if frameType == 0x03 {
+					if _, err = quicvarint.Read(buffer); err != nil { // Field: ECN Counts -> ECT0 Count
+						return nil, io.ErrUnexpectedEOF
+					}
+					if _, err = quicvarint.Read(buffer); err != nil { // Field: ECN Counts -> ECT1 Count
+						return nil, io.ErrUnexpectedEOF
+					}
+					if _, err = quicvarint.Read(buffer); err != nil { //nolint:misspell // Field: ECN Counts -> ECT-CE Count
+						return nil, io.ErrUnexpectedEOF
+					}
+				}
+			case 0x06: // CRYPTO frame, we will use this frame
+				offset, err := quicvarint.Read(buffer) // Field: Offset
+				if err != nil {
+					return nil, io.ErrUnexpectedEOF
+				}
+				length, err := quicvarint.Read(buffer) // Field: Length
+				if err != nil || length > uint64(buffer.Len()) {
+					return nil, io.ErrUnexpectedEOF
+				}
+				if cryptoLen < int(offset+length) {
+					cryptoLen = int(offset + length)
+					if len(cryptoData) < cryptoLen {
+						newCryptoData := bytespool.Alloc(int32(cryptoLen))
+						copy(newCryptoData, cryptoData)
+						bytespool.Free(cryptoData)
+						cryptoData = newCryptoData
+					}
+				}
+				if _, err := buffer.Read(cryptoData[offset : offset+length]); err != nil { // Field: Crypto Data
+					return nil, io.ErrUnexpectedEOF
+				}
+			case 0x1c: // CONNECTION_CLOSE frame, only 0x1c is permitted in initial packet
+				if _, err = quicvarint.Read(buffer); err != nil { // Field: Error Code
+					return nil, io.ErrUnexpectedEOF
+				}
+				if _, err = quicvarint.Read(buffer); err != nil { // Field: Frame Type
+					return nil, io.ErrUnexpectedEOF
+				}
+				length, err := quicvarint.Read(buffer) // Field: Reason Phrase Length
+				if err != nil {
+					return nil, io.ErrUnexpectedEOF
+				}
+				if _, err := buffer.ReadBytes(int32(length)); err != nil { // Field: Reason Phrase
+					return nil, io.ErrUnexpectedEOF
+				}
+			default:
+				// Only above frame types are permitted in initial packet.
+				// See https://www.rfc-editor.org/rfc/rfc9000.html#section-17.2.2-8
+				return nil, errNotQuicInitial
+			}
+		}
+
+		tlsHdr := &ptls.SniffHeader{}
+		err = ptls.ReadClientHello(cryptoData[:cryptoLen], tlsHdr)
+		if err != nil {
+			// The crypto data may have not been fully recovered in current packets,
+			// So we continue to sniff rest packets.
+			b = restPayload
+			continue
+		}
+		return &SniffHeader{domain: tlsHdr.Domain()}, nil
 	}
-	return &SniffHeader{domain: tlsHdr.Domain()}, nil
+	return nil, common.ErrNoClue
 }
 
 func hkdfExpandLabel(hash crypto.Hash, secret, context []byte, label string, length int) []byte {