Merge branch 'master' into adhoc

.gitignore

@@ -4,3 +4,4 @@ sessions/
 session.dat
 session.dat.new
 release/
+tdlib/

Dockerfile

@@ -29,7 +29,7 @@ WORKDIR /src
 RUN make ${MAKEOPTS}
 
 FROM scratch AS telegabber
-COPY --from=build /src/telegabber /usr/local/bin/
+COPY --from=build /src/release/telegabber /usr/local/bin/
 ENTRYPOINT ["/usr/local/bin/telegabber"]
 
 FROM scratch AS binaries

Makefile

@@ -6,7 +6,8 @@ VERSION := "v1.10.0-dev"
 MAKEOPTS := "-j4"
 
 all:
-	go build -ldflags "-X main.commit=${COMMIT}" -o telegabber
+	mkdir -p release
+	go build -ldflags "-X main.commit=${COMMIT}" -o release/telegabber
 
 test:
 	go test -v ./config ./ ./telegram ./xmpp ./xmpp/gateway ./persistence ./telegram/formatter ./badger
@@ -16,3 +17,9 @@ lint:
 
 build_indocker:
 	docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "VERSION=${VERSION}" --build-arg "MAKEOPTS=${MAKEOPTS}" --output=release --target binaries .
+
+build_indocker_staging:
+	DOCKER_BUILDKIT=1 docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "MAKEOPTS=${MAKEOPTS}" --network host --output=release --target binaries -f staging.Dockerfile .
+
+build_tdlib:
+	DOCKER_BUILDKIT=1 docker build --build-arg "TD_COMMIT=${TD_COMMIT}" --build-arg "MAKEOPTS=${MAKEOPTS}" --output=tdlib --target binaries -f tdlib.Dockerfile .

staging.Dockerfile

@@ -0,0 +1,46 @@
+FROM golang:1.19-bullseye AS base
+
+RUN apt-get update
+RUN apt-get install -y libssl-dev cmake build-essential gperf libz-dev make git php
+
+FROM base AS tdlib
+
+ARG TD_COMMIT
+ARG MAKEOPTS
+RUN git clone https://github.com/tdlib/td /src/
+RUN git -C /src/ checkout "${TD_COMMIT}"
+RUN mkdir build
+WORKDIR /build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/compiled/ /src/
+RUN cmake --build . --target prepare_cross_compiling ${MAKEOPTS}
+WORKDIR /src/
+RUN php SplitSource.php
+WORKDIR /build/
+RUN cmake --build . ${MAKEOPTS}
+RUN make install
+
+FROM base AS cache
+ARG VERSION
+COPY --from=tdlib /compiled/ /usr/local/
+WORKDIR /src
+RUN go env -w GOCACHE=/go-cache
+RUN go env -w GOMODCACHE=/gomod-cache
+RUN --mount=type=cache,target=/gomod-cache \
+    --mount=type=bind,source=./,target=/src \
+    go mod download
+
+FROM cache AS build
+ARG MAKEOPTS
+WORKDIR /src
+RUN --mount=type=bind,source=./,target=/src,rw \
+    --mount=type=cache,target=/go-cache \
+    --mount=type=cache,target=/gomod-cache \
+    --mount=type=cache,destination=/src/release \
+    make ${MAKEOPTS}
+
+FROM build AS release
+RUN --mount=type=cache,destination=/src/release \
+    cp /src/release/telegabber /
+
+FROM scratch AS binaries
+COPY --from=release /telegabber /

tdlib.Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.19-bullseye AS base
+
+RUN apt-get update
+RUN apt-get install -y libssl-dev cmake build-essential gperf libz-dev make git php
+
+FROM base AS tdlib
+
+ARG TD_COMMIT
+ARG MAKEOPTS
+RUN git clone https://github.com/tdlib/td /src/
+RUN git -C /src/ checkout "${TD_COMMIT}"
+RUN mkdir build
+WORKDIR /build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/compiled/ /src/
+RUN cmake --build . --target prepare_cross_compiling ${MAKEOPTS}
+WORKDIR /src/
+RUN php SplitSource.php
+WORKDIR /build/
+RUN cmake --build . ${MAKEOPTS}
+RUN make install
+
+FROM scratch AS binaries
+COPY --from=tdlib /compiled/ /

telegram/commands.go

@@ -362,16 +362,15 @@ func (c *Client) ProcessTransportCommand(cmdline string, resource string) (strin
 			return notOnline, false
 		}
 
-		for _, id := range c.cache.ChatsKeys() {
-			c.unsubscribe(id)
-		}
-
 		_, err := c.client.LogOut()
 		if err != nil {
-			c.forceClose()
 			return errors.Wrap(err, "Logout error").Error(), false
 		}
 
+		for _, id := range c.cache.ChatsKeys() {
+			c.unsubscribe(id)
+		}
+
 		c.Session.Login = ""
 	// cancel auth
 	case "cancelauth":
@@ -466,16 +465,6 @@ func (c *Client) ProcessTransportCommand(cmdline string, resource string) (strin
 			if gateway.MessageOutgoingPermissionVersion == 0 && args[0] == "carbons" && args[1] == "true" {
 				return "The server did not allow to enable carbons", false
 			}
-			if !c.Session.RawMessages && args[0] == "nativeedits" && args[1] == "true" {
-				return "nativeedits only works with rawmessages as of yet, enable it first", false
-			}
-			if c.Session.NativeEdits && args[0] == "rawmessages" && args[1] == "false" {
-				_, err := c.Session.Set("nativeedits", "false")
-				if err != nil {
-					return err.Error(), false
-				}
-				msg = "Automatically disabling nativeedits too...\n"
-			}
 
 			value, err := c.Session.Set(args[0], args[1])
 			if err != nil {

telegram/handlers.go

@@ -56,30 +56,28 @@ func (c *Client) cleanTempFile(path string) {
 }
 
 func (c *Client) sendMarker(chatId, messageId int64, typ gateway.MarkerType) {
-	if xmppId, err := gateway.IdsDB.GetByTgIds(c.Session.Login, c.jid, chatId, messageId); err == nil {
+	xmppId, err := gateway.IdsDB.GetByTgIds(c.Session.Login, c.jid, chatId, messageId)
-		resource := c.getFromOutbox(xmppId)
+	if err != nil {
-
+		xmppId = strconv.FormatInt(messageId, 10)
-		var stringType string
-		if typ == gateway.MarkerTypeReceived {
-			stringType = "received"
-		} else if typ == gateway.MarkerTypeDisplayed {
-			stringType = "displayed"
-		}
-		log.WithFields(log.Fields{
-			"xmppId":   xmppId,
-			"resource": resource,
-		}).Debugf("marker: %s", stringType)
-
-		if resource != "" {
-			gateway.SendMessageMarker(
-				c.jid+"/"+resource,
-				strconv.FormatInt(chatId, 10),
-				c.xmpp,
-				typ,
-				xmppId,
-			)
-		}
 	}
+
+	var stringType string
+	if typ == gateway.MarkerTypeReceived {
+		stringType = "received"
+	} else if typ == gateway.MarkerTypeDisplayed {
+		stringType = "displayed"
+	}
+	log.WithFields(log.Fields{
+		"xmppId": xmppId,
+	}).Debugf("marker: %s", stringType)
+
+	gateway.SendMessageMarker(
+		c.jid,
+		strconv.FormatInt(chatId, 10),
+		c.xmpp,
+		typ,
+		xmppId,
+	)
 }
 
 func (c *Client) updateHandler() {
@@ -306,38 +304,48 @@ func (c *Client) updateMessageContent(update *client.UpdateMessageContent) {
 			}
 			if ok && lastXmppId == xmppId {
 				replaceId = xmppId
-				message, err := c.client.GetMessage(&client.GetMessageRequest{
-					ChatId:    update.ChatId,
-					MessageId: update.MessageId,
-				})
-				if err == nil {
-					isCarbon = c.isCarbonsEnabled() && message.IsOutgoing
-				} else {
-					log.Errorf("No message %v/%v found, cannot reliably determine if it's a carbon", update.ChatId, update.MessageId)
-				}
 			} else {
 				log.Infof("Mismatching message ids: %v %v, falling back to separate edit message", lastXmppId, xmppId)
 			}
 		}
 
-		text := formatter.Format(
+		message, messageErr := c.client.GetMessage(&client.GetMessageRequest{
-			textContent.Text.Text,
+			ChatId:    update.ChatId,
-			textContent.Text.Entities,
+			MessageId: update.MessageId,
-			markupFunction,
+		})
-		)
+		var prefix string
+		if messageErr == nil {
+			isCarbon = c.isCarbonsEnabled() && message.IsOutgoing
+			// reply correction support in clients is suboptimal yet, so cut them out for now
+			prefix, _ = c.messageToPrefix(message, "", "", true)
+		} else {
+			log.Errorf("No message %v/%v found, cannot reliably determine if it's a carbon", update.ChatId, update.MessageId)
+		}
+
+		var text strings.Builder
 
 		if replaceId == "" {
 			var editChar string
 			if c.Session.AsciiArrows {
-				editChar = "e "
+				editChar = "e"
 			} else {
-				editChar = "✎ "
+				editChar = "✎"
 			}
-			text = editChar + fmt.Sprintf("%v | %s", update.MessageId, text)
+			text.WriteString(fmt.Sprintf("%s %v | ", editChar, update.MessageId))
+		} else if prefix != "" {
+			text.WriteString(prefix)
+			text.WriteString(c.getPrefixSeparator(update.ChatId))
 		}
 
+		text.WriteString(formatter.Format(
+			textContent.Text.Text,
+			textContent.Text.Entities,
+			markupFunction,
+		))
+
+		sChatId := strconv.FormatInt(update.ChatId, 10)
 		for _, jid := range jids {
-			gateway.SendMessage(jid, strconv.FormatInt(update.ChatId, 10), text, "e"+sId, c.xmpp, nil, replaceId, isCarbon, false)
+			gateway.SendMessage(jid, sChatId, text.String(), "e"+sId, c.xmpp, nil, replaceId, isCarbon, false)
 		}
 	}
 }
@@ -373,6 +381,8 @@ func (c *Client) updateMessageSendSucceeded(update *client.UpdateMessageSendSucc
 		log.Errorf("failed to replace %v with %v: %v", update.OldMessageId, update.Message.Id, err.Error())
 	}
 
+	c.updateLastMessageHash(update.Message.ChatId, update.Message.Id, update.Message.Content)
+
 	c.sendMarker(update.Message.ChatId, update.Message.Id, gateway.MarkerTypeReceived)
 
 	// clean uploaded files

telegram/utils.go

@@ -967,7 +967,7 @@ func (c *Client) isCarbonsEnabled() bool {
 	return gateway.MessageOutgoingPermissionVersion > 0 && c.Session.Carbons
 }
 
-func (c *Client) messageToPrefix(message *client.Message, previewString string, fileString string) (string, *gateway.Reply) {
+func (c *Client) messageToPrefix(message *client.Message, previewString string, fileString string, suppressReply bool) (string, *gateway.Reply) {
 	isPM, err := c.IsPM(message.ChatId)
 	if err != nil {
 		log.Errorf("Could not determine if chat is PM: %v", err)
@@ -1005,27 +1005,32 @@ func (c *Client) messageToPrefix(message *client.Message, previewString string,
 	}
 
 	// reply to
-	preview := true
+	var reply *gateway.Reply
-	reply, tgReply := c.getMessageReply(message, preview, false)
+	if !suppressReply {
+		preview := true
+		gwReply, tgReply := c.getMessageReply(message, preview, false)
 
-	if tgReply != nil {
+		if tgReply != nil {
-		var replyStart, replyEnd int
+			reply = gwReply
 
-		if len(prefix) > 0 {
+			var replyStart, replyEnd int
-			replyStart = c.countCharsInLines(&prefix) + (len(prefix)-1)*len(messageHeaderSeparator)
-		}
 
-		replyLine := "reply: " + c.formatMessageContent(preview, tgReply)
+			if len(prefix) > 0 {
-		prefix = append(prefix, replyLine)
+				replyStart = c.countCharsInLines(&prefix) + (len(prefix)-1)*len(messageHeaderSeparator)
+			}
 
-		replyEnd = replyStart + utf8.RuneCountInString(replyLine)
+			replyLine := "reply: " + c.formatMessageContent(preview, tgReply)
-		if len(prefix) > 0 {
+			prefix = append(prefix, replyLine)
-			replyEnd += len(messageHeaderSeparator)
-		}
 
-		if reply != nil {
+			replyEnd = replyStart + utf8.RuneCountInString(replyLine)
-			reply.Start = uint64(replyStart)
+			if len(prefix) > 0 {
-			reply.End = uint64(replyEnd)
+				replyEnd += len(messageHeaderSeparator)
+			}
+
+			if reply != nil {
+				reply.Start = uint64(replyStart)
+				reply.End = uint64(replyEnd)
+			}
 		}
 	}
 
@@ -1060,6 +1065,17 @@ func (c *Client) ensureDownloadFile(file *client.File) *client.File {
 	return file
 }
 
+// \n if it is groupchat and message is not empty
+func (c *Client) getPrefixSeparator(chatId int64) string {
+	var separator string
+	if chatId < 0 {
+		separator = "\n"
+	} else if chatId > 0 {
+		separator = " | "
+	}
+	return separator
+}
+
 // ProcessIncomingMessage transfers a message to XMPP side and marks it as read on Telegram side
 func (c *Client) ProcessIncomingMessage(chatId int64, message *client.Message) {
 	isCarbon := c.isCarbonsEnabled() && message.IsOutgoing
@@ -1103,21 +1119,15 @@ func (c *Client) ProcessIncomingMessage(chatId int64, message *client.Message) {
 			} else if !c.Session.RawMessages {
 				var newText strings.Builder
 
-				prefix, prefixReply := c.messageToPrefix(message, previewName, fileName)
+				prefix, prefixReply := c.messageToPrefix(message, previewName, fileName, false)
 				reply = prefixReply
 				replyObtained = true
 				newText.WriteString(prefix)
 
 				if text != "" {
-					// \n if it is groupchat and message is not empty
 					if prefix != "" {
-						if chatId < 0 {
+						newText.WriteString(c.getPrefixSeparator(chatId))
-							newText.WriteString("\n")
-						} else if chatId > 0 {
-							newText.WriteString(" | ")
-						}
 					}
-
 					newText.WriteString(text)
 				}
 				text = newText.String()

telegram/utils_test.go

@@ -436,7 +436,7 @@ func TestMessageToPrefix1(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "", "", false)
 	if prefix != "➡ 42 | fwd: ziz" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -454,7 +454,7 @@ func TestMessageToPrefix2(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "y.jpg", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{}}).messageToPrefix(&message, "y.jpg", "", false)
 	if prefix != "⬅ 56 | fwd:  (zaz) | preview: y.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -472,7 +472,7 @@ func TestMessageToPrefix3(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "a.jpg")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "a.jpg", false)
 	if prefix != "< 56 | fwd:  (zuz) | file: a.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -486,7 +486,7 @@ func TestMessageToPrefix4(t *testing.T) {
 		Id:         23,
 		IsOutgoing: true,
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -504,7 +504,7 @@ func TestMessageToPrefix5(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "h.jpg", "a.jpg")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "h.jpg", "a.jpg", false)
 	if prefix != "< 560 | fwd:  (zyz) | preview: h.jpg | file: a.jpg" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -530,7 +530,7 @@ func TestMessageToPrefix6(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23 | reply: ziz @ unknown contact: TDlib instance is offline | tist uz iz" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -556,7 +556,7 @@ func TestMessageToPrefix7(t *testing.T) {
 			},
 		},
 	}
-	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "")
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", false)
 	if prefix != "> 23 | reply:  (zaz) @ unknown contact: TDlib instance is offline | tist" {
 		t.Errorf("Wrong prefix: %v", prefix)
 	}
@@ -565,6 +565,32 @@ func TestMessageToPrefix7(t *testing.T) {
 	}
 }
 
+func TestMessageToPrefix8(t *testing.T) {
+	message := client.Message{
+		Id:         23,
+		ChatId:     42,
+		IsOutgoing: true,
+		ReplyTo: &client.MessageReplyToMessage{
+			ChatId: 41,
+			Content: &client.MessageText{
+				Text: &client.FormattedText{
+					Text: "tist",
+				},
+			},
+			Origin: &client.MessageOriginChannel{
+				AuthorSignature: "zuz",
+			},
+		},
+	}
+	prefix, gatewayReply := (&Client{Session: &persistence.Session{AsciiArrows: true}}).messageToPrefix(&message, "", "", true)
+	if prefix != "> 23" {
+		t.Errorf("Wrong prefix: %v", prefix)
+	}
+	if gatewayReply != nil {
+		t.Errorf("Reply is not nil: %v", gatewayReply)
+	}
+}
+
 func GetSenderIdEmpty(t *testing.T) {
 	message := client.Message{}
 	senderId := (&Client{}).getMessageSenderId(&message)

xmpp/handlers.go

@@ -210,7 +210,7 @@ func HandleMessage(s xmpp.Sender, p stanza.Packet) {
 				} else {
 					err = gateway.IdsDB.Set(session.Session.Login, bare, toID, tgMessageId, msg.Id)
 					if err == nil {
-						session.AddToOutbox(msg.Id, resource)
+						// session.AddToOutbox(msg.Id, resource)
 						session.UpdateLastChatMessageId(toID, msg.Id)
 					} else {
 						log.Errorf("Failed to save ids %v/%v %v", toID, tgMessageId, msg.Id)