Read Chrome Web Store reviews

docs/decisions/0009-reviews.md

@@ -22,9 +22,15 @@ reviews on a separate page, without a stable ID per review.
   review that matters.
 - `read_at` is the user's: `PATCH /api/reviews/{id}` sets it. A review whose stars or
   text change becomes unread again.
-- `GET /api/reviews` filters by unread, unanswered (Marketplace reviews without a
-  reply), stars, store and extension, newest first, with `limit`/`offset` and a total.
-- Chrome Web Store reviews are not read yet (`fetchReviews` returns none).
+- `GET /api/reviews` filters by unread, unanswered (reviews without a reply, on the
+  stores that have replies: not Open VSX), stars, store and extension, newest first,
+  with `limit`/`offset` and a total.
+- Chrome Web Store reviews (added 2026-10-28) come from the item's reviews page, which
+  carries them as JSON for its scripts (`AF_initDataCallback`), each with a review ID,
+  stars, text, version, last-edit time and the developer's reply. The page lists the 10
+  latest reviews in its language, so `extpulse.stores.chrome-web-store-review-languages`
+  (English and Portuguese) is one request each. Ratings without a comment aren't listed
+  by the store.
 - If reading reviews fails, the day's snapshot is still saved; the listing is counted as
   failed with a `reviews:` message.
 
@@ -34,13 +40,18 @@ reviews on a separate page, without a stable ID per review.
   as long (12 listings: about 12 s).
 - A deleted review stays in the table as it was last seen; the store's deletions are
   not mirrored yet.
-- Only the latest 100 reviews per listing are read each day, which is enough unless a
-  listing gets more than 100 in a day.
+- Only the latest 100 reviews per listing are read each day (10 per language on the
+  Chrome Web Store), which is enough unless a listing gets more than that in a day.
+  Reviews in a language that isn't configured are not read.
+- The Chrome Web Store adds one request per configured language per listing.
 
 ## Alternatives considered
 
-- **Reading Chrome reviews now:** possible from the page, but with no review ID the key
-  would be author + date, which breaks when a review is edited. Left until there is a
-  Chrome extension with written reviews to test against.
+- **Reading Chrome reviews from the visible text:** there were no stars and no review ID
+  in it, so the key would have been author + date, which breaks when a review is
+  edited. The JSON the page carries has both.
+- **The store's internal data API (`batchexecute`):** returns 100 reviews per call, but
+  it is an undocumented endpoint for the site's own scripts; the page is what any
+  visitor gets.
 - **A JPA entity for reviews:** the writes are upserts and the reads are filtered joins
   across four tables; plain SQL says both more clearly.

src/main/java/com/mccprogrammer/extpulse/reviews/ReviewController.java

@@ -26,8 +26,8 @@
 import org.springframework.web.bind.annotation.RestController;
 
 /**
- * One inbox for the reviews of every store (the Marketplace and Open VSX; the Chrome Web
- * Store's are not read yet).
+ * One inbox for the reviews of every store: the Marketplace, Open VSX and the Chrome Web
+ * Store.
  */
 @RestController
 class ReviewController {
@@ -44,8 +44,8 @@ class ReviewController {
 	/**
 	 * Your reviews, newest first.
 	 * @param unread only unread (true) or only read (false) reviews
-	 * @param unanswered only Marketplace reviews without your reply (Open VSX has no
-	 * replies)
+	 * @param unanswered only reviews without your reply, on the stores that have replies
+	 * (not Open VSX)
 	 * @param maxRating only reviews with at most this many stars
 	 */
 	@GetMapping("/api/reviews")
@@ -84,7 +84,7 @@ record ReviewResponse(long id, long extensionId, String extensionName, Store sto
 			Instant firstSeenAt) {
 
 		static ReviewResponse of(ReviewRow row) {
-			Boolean replied = (row.store() == Store.VS_MARKETPLACE) ? row.reply() != null : null;
+			Boolean replied = row.store().hasReplies() ? row.reply() != null : null;
 			return new ReviewResponse(row.id(), row.extensionId(), row.extensionName(), row.store(), row.rating(),
 					row.text(), row.author(), row.productVersion(), row.postedAt().toInstant(), row.reply(), replied,
 					row.readAt() != null, row.firstSeenAt().toInstant());

src/main/java/com/mccprogrammer/extpulse/reviews/ReviewQueries.java

@@ -36,7 +36,7 @@ List<ReviewRow> inbox(long userId, Filter filter, int limit, int offset) {
 				JOIN publishers p ON p.id = e.publisher_id
 				WHERE p.user_id = :user
 				  AND (CAST(:unread AS boolean) IS NULL OR (r.read_at IS NULL) = CAST(:unread AS boolean))
-				  AND (CAST(:unanswered AS boolean) IS NOT TRUE OR (l.store = 'VS_MARKETPLACE' AND r.reply IS NULL))
+				  AND (CAST(:unanswered AS boolean) IS NOT TRUE OR (l.store <> 'OPEN_VSX' AND r.reply IS NULL))
 				  AND (CAST(:maxRating AS integer) IS NULL OR r.rating <= CAST(:maxRating AS integer))
 				  AND (CAST(:store AS text) IS NULL OR l.store = CAST(:store AS text))
 				  AND (CAST(:extension AS bigint) IS NULL OR e.id = CAST(:extension AS bigint))

src/main/java/com/mccprogrammer/extpulse/store/ChromeWebStoreSource.java

@@ -3,13 +3,21 @@
 import java.math.BigDecimal;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Comparator;
 import java.util.HexFormat;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import tools.jackson.core.JacksonException;
+import tools.jackson.databind.JsonNode;
+import tools.jackson.databind.json.JsonMapper;
 
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestClient;
@@ -21,6 +29,11 @@
  * CSS classes, which are generated and change with every release of the site. When the
  * page changes anyway, this throws a {@link StoreException} instead of guessing; the
  * weekly live-stores workflow is there to notice.
+ * <p>
+ * Reviews are on another page, /detail/{id}/reviews, which carries them as data for its
+ * scripts: an {@code AF_initDataCallback} block of JSON arrays, the same data the page
+ * shows. It lists the 10 latest reviews in the page's language, so each configured
+ * language is one request.
  */
 @Component
 class ChromeWebStoreSource implements StoreSource {
@@ -34,6 +47,32 @@ class ChromeWebStoreSource implements StoreSource {
 
 	private static final Pattern VERSION = Pattern.compile("Version (\\S+) Updated");
 
+	// The page's data requests: the reviews are the one asking for the item and a page
+	// size, as in 'ds:1' : {id:'x1DgCd',request:["{item}",[10],...]}.
+	private static final Pattern REVIEWS_REQUEST = Pattern
+		.compile("'(ds:\\d+)'\\s*:\\s*\\{id:'[^']+',request:\\[\"([a-p]{32})\",\\[\\d+]");
+
+	// Where each field of a review is in its array.
+	private static final int REVIEW_ID = 0;
+
+	private static final int REVIEW_AUTHOR = 1;
+
+	private static final int REVIEW_RATING = 2;
+
+	private static final int REVIEW_TEXT = 3;
+
+	private static final int REVIEW_UPDATED = 5;
+
+	private static final int REVIEW_REPLY = 8;
+
+	private static final int REVIEW_VERSION = 11;
+
+	private static final int REPLY_TEXT = 2;
+
+	private static final int MAX_REVIEWS = 100;
+
+	private static final JsonMapper JSON = JsonMapper.builder().build();
+
 	// From the EU there are three: cookie consent, back to the store, then the item's
 	// name.
 	private static final int MAX_REDIRECTS = 5;
@@ -42,9 +81,12 @@ class ChromeWebStoreSource implements StoreSource {
 
 	private final String baseUrl;
 
+	private final List<String> reviewLanguages;
+
 	ChromeWebStoreSource(StoreProperties properties) {
 		this.http = StoreHttp.client(properties.chromeWebStoreUrl(), properties);
 		this.baseUrl = properties.chromeWebStoreUrl().toString();
+		this.reviewLanguages = properties.chromeWebStoreReviewLanguages();
 	}
 
 	@Override
@@ -62,24 +104,39 @@ public List<DiscoveredListing> discover(String publisher) {
 	}
 
 	/**
-	 * Always empty for now. The reviews are on a separate page (/detail/{id}/reviews)
-	 * with no stable ID per review; reading them is left for later.
+	 * The latest reviews in each configured language, newest first. Ratings without a
+	 * comment aren't listed by the store, so they aren't here either.
 	 */
 	@Override
 	public List<StoreReview> fetchReviews(String externalId) {
-		return List.of();
+		checkItemId(externalId);
+		Map<String, StoreReview> reviews = new LinkedHashMap<>();
+		for (String language : this.reviewLanguages) {
+			for (StoreReview review : parseReviews(externalId, page(externalId, "/reviews?hl=" + language))) {
+				reviews.putIfAbsent(review.externalId(), review);
+			}
+		}
+		return reviews.values()
+			.stream()
+			.sorted(Comparator.comparing(StoreReview::postedAt).reversed())
+			.limit(MAX_REVIEWS)
+			.toList();
 	}
 
 	@Override
 	public ListingStats fetchStats(String externalId) {
+		checkItemId(externalId);
+		return parse(externalId, page(externalId, "?hl=en"));
+	}
+
+	private static void checkItemId(String externalId) {
 		if (!ITEM_ID.matcher(externalId).matches()) {
 			throw new IllegalArgumentException("Not a Chrome Web Store item ID: " + externalId);
 		}
-		return parse(externalId, page(externalId));
 	}
 
-	private String page(String externalId) {
-		URI uri = URI.create(this.baseUrl + "/detail/" + externalId + "?hl=en");
+	private String page(String externalId, String rest) {
+		URI uri = URI.create(this.baseUrl + "/detail/" + externalId + rest);
 		try {
 			for (int redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
 				URI current = uri;
@@ -146,6 +203,66 @@ private ListingStats parse(String externalId, String html) {
 				rated ? (int) number(rating.group(3)) : 0, version.find() ? version.group(1) : null);
 	}
 
+	private List<StoreReview> parseReviews(String externalId, String html) {
+		Document page = Jsoup.parse(html);
+		if (page.selectFirst("h1") == null) {
+			throw new ListingNotFoundException(store(), externalId);
+		}
+		JsonNode data = reviewsData(page, externalId);
+		JsonNode reviews = data.path(1);
+		if (!reviews.isArray()) {
+			// [[]]: the store has no review with a comment in this language.
+			return List.of();
+		}
+		return reviews.valueStream().map((review) -> review(externalId, review)).toList();
+	}
+
+	/**
+	 * The JSON the page holds for its reviews: [next page token, [review, ...], total].
+	 */
+	private JsonNode reviewsData(Document page, String externalId) {
+		String key = null;
+		for (Element script : page.select("script")) {
+			Matcher request = REVIEWS_REQUEST.matcher(script.data());
+			if (request.find() && request.group(2).equals(externalId)) {
+				key = request.group(1);
+			}
+		}
+		String callback = "AF_initDataCallback({key: '" + key + "'";
+		for (Element script : page.select("script")) {
+			String code = script.data();
+			int start = code.indexOf("data:");
+			int end = code.lastIndexOf(", sideChannel:");
+			if (key != null && code.startsWith(callback) && start > 0 && end > start) {
+				try {
+					return JSON.readTree(code.substring(start + "data:".length(), end));
+				}
+				catch (JacksonException ex) {
+					throw new StoreException(store(), "unreadable reviews on the page of " + externalId, ex);
+				}
+			}
+		}
+		throw new StoreException(store(), "no reviews data on the page of " + externalId + "; has the page changed?");
+	}
+
+	private StoreReview review(String externalId, JsonNode review) {
+		String id = review.path(REVIEW_ID).asString("");
+		int rating = review.path(REVIEW_RATING).asInt(0);
+		JsonNode updated = review.path(REVIEW_UPDATED);
+		if (id.isEmpty() || rating < 1 || rating > 5 || !updated.path(0).isNumber()) {
+			throw new StoreException(store(),
+					"a review on the page of " + externalId + " isn't as expected: " + review);
+		}
+		return new StoreReview(id, rating, textOrNull(review.path(REVIEW_TEXT)),
+				textOrNull(review.path(REVIEW_AUTHOR).path(0)), textOrNull(review.path(REVIEW_VERSION)),
+				Instant.ofEpochSecond(updated.path(0).asLong(), updated.path(1).asLong(0)),
+				textOrNull(review.path(REVIEW_REPLY).path(REPLY_TEXT)));
+	}
+
+	private static String textOrNull(JsonNode node) {
+		return (node.isString() && !node.asString().isBlank()) ? node.asString() : null;
+	}
+
 	private Matcher find(Pattern pattern, String text, String externalId, String what) {
 		Matcher matcher = pattern.matcher(text);
 		if (!matcher.find()) {

src/main/java/com/mccprogrammer/extpulse/store/Store.java

@@ -23,4 +23,11 @@ public String displayName() {
 		return this.displayName;
 	}
 
+	/**
+	 * Whether the publisher can reply to reviews on this store (Open VSX has no replies).
+	 */
+	public boolean hasReplies() {
+		return this != OPEN_VSX;
+	}
+
 }

src/main/java/com/mccprogrammer/extpulse/store/StoreProperties.java

@@ -2,14 +2,25 @@
 
 import java.net.URI;
 import java.time.Duration;
+import java.util.List;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
 
 /**
  * Where the stores are and how long to wait for them ({@code extpulse.stores.*} in
  * application.yaml). Tests point the URLs at WireMock.
+ *
+ * @param chromeWebStoreReviewLanguages the languages whose Chrome Web Store reviews are
+ * read: the store shows the reviews of one language at a time
  */
 @ConfigurationProperties("extpulse.stores")
 public record StoreProperties(URI marketplaceUrl, URI openVsxUrl, URI chromeWebStoreUrl, Duration connectTimeout,
-		Duration readTimeout) {
+		Duration readTimeout, List<String> chromeWebStoreReviewLanguages) {
+
+	public StoreProperties {
+		chromeWebStoreReviewLanguages = (chromeWebStoreReviewLanguages == null
+				|| chromeWebStoreReviewLanguages.isEmpty()) ? List.of("en")
+						: List.copyOf(chromeWebStoreReviewLanguages);
+	}
+
 }

src/main/java/com/mccprogrammer/extpulse/store/StoreReview.java

@@ -6,12 +6,13 @@
  * A review as a store shows it.
  *
  * @param externalId what identifies the review on its listing: the review ID on the
- * Marketplace; on Open VSX, which has no review IDs and allows one review per user, the
- * author ({@code github:login})
+ * Marketplace and the Chrome Web Store; on Open VSX, which has no review IDs and allows
+ * one review per user, the author ({@code github:login})
  * @param text the comment, or {@code null} for a rating without one
  * @param productVersion the version reviewed, if the store says
  * @param postedAt when it was written or last edited
- * @param reply the publisher's reply, on stores that have replies (the Marketplace)
+ * @param reply the publisher's reply, on stores that have replies (the Marketplace and
+ * the Chrome Web Store)
  */
 public record StoreReview(String externalId, int rating, String text, String author, String productVersion,
 		Instant postedAt, String reply) {

src/main/resources/application.yaml

@@ -71,6 +71,9 @@ extpulse:
     marketplace-url: https://marketplace.visualstudio.com
     open-vsx-url: https://open-vsx.org
     chrome-web-store-url: https://chromewebstore.google.com
+    # The Chrome Web Store lists the reviews of one language at a time (the page's
+    # language): one request each, every day.
+    chrome-web-store-review-languages: [en, pt]
     # A store that doesn't answer in time fails the request instead of hanging.
     connect-timeout: 5s
     read-timeout: 10s

src/main/resources/templates/reviews/inbox.html

@@ -7,7 +7,7 @@
 	<header th:replace="~{fragments :: topbar('reviews')}"></header>
 	<main>
 		<h1>Reviews</h1>
-		<p class="lede">From the VS Code Marketplace and Open VSX, newest first.</p>
+		<p class="lede">From the VS Code Marketplace, Open VSX and the Chrome Web Store, newest first.</p>
 		<div th:replace="~{fragments :: flash}"></div>
 
 		<form class="row card" method="get" action="/app/reviews" aria-label="Filters">

src/test/java/com/mccprogrammer/extpulse/reviews/ReviewsApiTests.java

@@ -92,6 +92,17 @@ void filtersByStarsStoreExtensionAndReply() throws Exception {
 		inbox("?unanswered=true").andExpect(jsonPath("$.items[*].text").value(contains("nice", "great")));
 	}
 
+	@Test
+	void chromeWebStoreReviewsHaveRepliesToo() throws Exception {
+		long chrome = listing(this.markify, "CHROME_WEB_STORE", "kpjlaobgikkhpngdkladodopdkigmdei");
+		this.writer
+			.save(chrome, List.of(review("fa09b0ff-8d4a-4f16-a49f-44ae1928850f", 3, "ok", "2026-06-01T10:00:00Z", null),
+					review("c973bfb1-c5fe-43bd-a50b-e6acf49f3a8c", 5, "love it", "2026-05-15T10:00:00Z", "Thanks!")));
+
+		inbox("?store=CHROME_WEB_STORE").andExpect(jsonPath("$.items[*].replied").value(contains(false, true)));
+		inbox("?unanswered=true").andExpect(jsonPath("$.items[*].text").value(contains("ok", "nice", "great")));
+	}
+
 	@Test
 	void marksReviewsAsReadAndFiltersThem() throws Exception {
 		long newest = this.jdbc.sql("SELECT id FROM reviews ORDER BY posted_at DESC LIMIT 1")