Merge branch 'maint-0.3.5' into maint-0.4.0

changes/bug30916

@@ -0,0 +1,4 @@
+  o Minor bugfixes (relay):
+    - Avoid crashing when starting with a corrupt keys directory where
+      the old ntor key and the new ntor key are identical. Fixes bug 30916;
+      bugfix on 0.2.4.8-alpha.

changes/bug31107

@@ -0,0 +1,4 @@
+  o Minor bugfixes (logging, protocol violations):
+    - Do not log a nonfatal assertion failure when receiving a VERSIONS
+      cell on a connection using the obsolete v1 link protocol. Log a
+      protocol_warn instead. Fixes bug 31107; bugfix on 0.2.4.4-alpha.

changes/bug31408

@@ -0,0 +1,5 @@
+  o Major bugfixes (torrc):
+    - Stop ignoring torrc options after an %include directive, when the
+      included directory ends with a file that does not contain any config
+      options. (But does contain comments or whitespace.)
+      Fixes bug 31408; bugfix on 0.3.1.1-alpha.

changes/bug31837

@@ -0,0 +1,5 @@
+  o Minor bugfixes (testing):
+    - When testing port rebinding, don't busy-wait for tor to log. Instead,
+      actually sleep for a short time before polling again. Also improve the
+      formatting of control commands and log messages.
+      Fixes bug 31837; bugfix on 0.3.5.1-alpha.

changes/ticket31466

@@ -0,0 +1,5 @@
+  o Minor bugfixes (logging):
+    - Rate-limit our the logging message about the obsolete .exit notation.
+      Previously, there was no limit on this warning, which could potentially
+      be triggered many times by a hostile website. Fixes bug 31466;
+      bugfix on 0.2.2.1-alpha.

src/core/or/channeltls.c

@@ -1106,7 +1106,15 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
       /* do nothing */
       break;
     case CELL_VERSIONS:
-      tor_fragile_assert();
+      /* A VERSIONS cell should always be a variable-length cell, and
+       * so should never reach this function (which handles constant-sized
+       * cells). But if the connection is using the (obsolete) v1 link
+       * protocol, all cells will be treated as constant-sized, and so
+       * it's possible we'll reach this code.
+       */
+      log_fn(LOG_PROTOCOL_WARN, LD_CHANNEL,
+             "Received unexpected VERSIONS cell on a channel using link "
+             "protocol %d; ignoring.", conn->link_proto);
       break;
     case CELL_NETINFO:
       ++stats_n_netinfo_cells_processed;

src/core/or/connection_edge.c

@@ -1610,8 +1610,10 @@ connection_ap_handshake_rewrite(entry_connection_t *conn,
    * disallowed when they're coming straight from the client, but you're
    * allowed to have them in MapAddress commands and so forth. */
   if (!strcmpend(socks->address, ".exit")) {
-    log_warn(LD_APP, "The  \".exit\" notation is disabled in Tor due to "
-             "security risks.");
+    static ratelim_t exit_warning_limit = RATELIM_INIT(60*15);
+    log_fn_ratelim(&exit_warning_limit, LOG_WARN, LD_APP,
+                   "The  \".exit\" notation is disabled in Tor due to "
+                   "security risks.");
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
     out->end_reason = END_STREAM_REASON_TORPROTOCOL;

src/feature/relay/router.c

@@ -281,19 +281,17 @@ construct_ntor_key_map(void)
 {
   di_digest256_map_t *m = NULL;
 
-  if (!tor_mem_is_zero((const char*)
-                       curve25519_onion_key.pubkey.public_key,
-                       CURVE25519_PUBKEY_LEN)) {
-    dimap_add_entry(&m,
-                    curve25519_onion_key.pubkey.public_key,
+  const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
+  const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
+
+  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m, cur_pk,
                     tor_memdup(&curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
-  if (!tor_mem_is_zero((const char*)
-                          last_curve25519_onion_key.pubkey.public_key,
-                       CURVE25519_PUBKEY_LEN)) {
-    dimap_add_entry(&m,
-                    last_curve25519_onion_key.pubkey.public_key,
+  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+      tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
+    dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }

src/lib/fs/conffile.c

@@ -153,16 +153,18 @@ config_process_include(const char *path, int recursion_level, int extended,
   int rv = -1;
   SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
     config_line_t *included_config = NULL;
+    config_line_t *included_config_last = NULL;
     if (config_get_included_config(config_file, recursion_level, extended,
-                                   &included_config, list_last,
+                                   &included_config, &included_config_last,
                                    opened_lst) < 0) {
       goto done;
     }
 
     *next = included_config;
-    if (*list_last)
-      next = &(*list_last)->next;
-
+    if (included_config_last) {
+      next = &included_config_last->next;
+      *list_last = included_config_last;
+    }
   } SMARTLIST_FOREACH_END(config_file);
   *list = ret_list;
   rv = 0;

src/test/test_config.c

@@ -5287,6 +5287,73 @@ test_config_include_folder_order(void *data)
   tor_free(dir);
 }
 
+static void
+test_config_include_blank_file_last(void *data)
+{
+  (void)data;
+
+  config_line_t *result = NULL;
+  char *torrcd = NULL;
+  char *path = NULL;
+  char *dir = tor_strdup(get_fname("test_include_blank_file_last"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&torrcd, "%s"PATH_SEPARATOR"%s", dir, "torrc.d");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(torrcd), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(torrcd, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "aa_1st");
+  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "bb_2nd");
+  tt_int_op(write_str_to_file(path, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", torrcd, "cc_comment");
+  tt_int_op(write_str_to_file(path, "# comment only\n", 0), OP_EQ, 0);
+  tor_free(path);
+
+  char torrc_contents[1000];
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s\n"
+               "Test 3\n",
+               torrcd);
+
+  int include_used;
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  for (next = result; next != NULL; next = next->next) {
+    char expected[10];
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 3);
+
+ done:
+  config_free_lines(result);
+  tor_free(torrcd);
+  tor_free(path);
+  tor_free(dir);
+}
+
 static void
 test_config_include_path_syntax(void *data)
 {
@@ -5927,6 +5994,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_recursion_before_after, 0),
   CONFIG_TEST(include_recursion_after_only, 0),
   CONFIG_TEST(include_folder_order, 0),
+  CONFIG_TEST(include_blank_file_last, 0),
   CONFIG_TEST(include_path_syntax, 0),
   CONFIG_TEST(include_not_processed, 0),
   CONFIG_TEST(include_has_include, 0),

src/test/test_rebind.py

@@ -32,15 +32,17 @@ def wait_for_log(s):
     cutoff = time.time() + LOG_TIMEOUT
     while time.time() < cutoff:
         l = tor_process.stdout.readline()
-        l = l.decode('utf8')
+        l = l.decode('utf8', 'backslashreplace')
         if s in l:
             logging.info('Tor logged: "{}"'.format(l.strip()))
             return
-        logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
         # readline() returns a blank string when there is no output
         # avoid busy-waiting
-        if len(s) == 0:
+        if len(l) == 0:
+            logging.debug('Tor has not logged anything, waiting for "{}"'.format(s))
             time.sleep(LOG_WAIT)
+        else:
+            logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
     fail('Could not find "{}" in logs after {} seconds'.format(s, LOG_TIMEOUT))
 
 def pick_random_port():
@@ -120,18 +122,18 @@ def pick_random_port():
     tor_process.terminate()
     fail('Cannot connect to ControlPort')
 
-control_socket.sendall('AUTHENTICATE \r\n'.encode('utf8'))
-control_socket.sendall('SETCONF SOCKSPort=0.0.0.0:{}\r\n'.format(socks_port).encode('utf8'))
+control_socket.sendall('AUTHENTICATE \r\n'.encode('ascii'))
+control_socket.sendall('SETCONF SOCKSPort=0.0.0.0:{}\r\n'.format(socks_port).encode('ascii'))
 wait_for_log('Opened Socks listener')
 
 try_connecting_to_socksport()
 
-control_socket.sendall('SETCONF SOCKSPort=127.0.0.1:{}\r\n'.format(socks_port).encode('utf8'))
+control_socket.sendall('SETCONF SOCKSPort=127.0.0.1:{}\r\n'.format(socks_port).encode('ascii'))
 wait_for_log('Opened Socks listener')
 
 try_connecting_to_socksport()
 
-control_socket.sendall('SIGNAL HALT\r\n'.encode('utf8'))
+control_socket.sendall('SIGNAL HALT\r\n'.encode('ascii'))
 
 wait_for_log('exiting cleanly')
 logging.info('OK')