gh-107211: No longer export internal functions (7) (#108425)

No longer export _PyUnicode_FromId() internal C API function.

Change comment style to "// comment" and add comment explaining why
other functions have to be exported.

Update Tools/build/generate_token.py to update Include/internal/pycore_token.h
comments.

Include/internal/pycore_unicodeobject.h

@@ -24,6 +24,7 @@ extern int _PyUnicode_IsCased(Py_UCS4 ch);
 
 /* --- Unicode API -------------------------------------------------------- */
 
+// Export for '_json' shared extension
 PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
     PyObject *op,
     int check_content);
@@ -31,10 +32,10 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
 extern void _PyUnicode_ExactDealloc(PyObject *op);
 extern Py_ssize_t _PyUnicode_InternedSize(void);
 
-/* Get a copy of a Unicode string. */
+// Get a copy of a Unicode string.
+// Export for '_datetime' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_Copy(
-    PyObject *unicode
-    );
+    PyObject *unicode);
 
 /* Unsafe version of PyUnicode_Fill(): don't check arguments and so may crash
    if parameters are invalid (e.g. if length is longer than the string). */
@@ -93,11 +94,13 @@ typedef struct {
     unsigned char readonly;
 } _PyUnicodeWriter ;
 
-/* Initialize a Unicode writer.
- *
- * By default, the minimum buffer size is 0 character and overallocation is
- * disabled. Set min_length, min_char and overallocate attributes to control
- * the allocation of the buffer. */
+// Initialize a Unicode writer.
+//
+// By default, the minimum buffer size is 0 character and overallocation is
+// disabled. Set min_length, min_char and overallocate attributes to control
+// the allocation of the buffer.
+//
+// Export the _PyUnicodeWriter API for '_multibytecodec' shared extension.
 PyAPI_FUNC(void)
 _PyUnicodeWriter_Init(_PyUnicodeWriter *writer);
 
@@ -204,12 +207,14 @@ extern PyObject* _PyUnicode_EncodeUTF7(
 
 /* --- UTF-8 Codecs ------------------------------------------------------- */
 
+// Export for '_tkinter' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_AsUTF8String(
     PyObject *unicode,
     const char *errors);
 
 /* --- UTF-32 Codecs ------------------------------------------------------ */
 
+// Export for '_tkinter' shared extension
 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
     PyObject *object,           /* Unicode object */
     const char *errors,         /* error handling */
@@ -217,20 +222,21 @@ PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF32(
 
 /* --- UTF-16 Codecs ------------------------------------------------------ */
 
-/* Returns a Python string object holding the UTF-16 encoded value of
-   the Unicode data.
-
-   If byteorder is not 0, output is written according to the following
-   byte order:
-
-   byteorder == -1: little endian
-   byteorder == 0:  native byte order (writes a BOM mark)
-   byteorder == 1:  big endian
-
-   If byteorder is 0, the output string will always start with the
-   Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
-   prepended.
-*/
+// Returns a Python string object holding the UTF-16 encoded value of
+// the Unicode data.
+//
+// If byteorder is not 0, output is written according to the following
+// byte order:
+//
+// byteorder == -1: little endian
+// byteorder == 0:  native byte order (writes a BOM mark)
+// byteorder == 1:  big endian
+//
+// If byteorder is 0, the output string will always start with the
+// Unicode BOM mark (U+FEFF). In the other two modes, no BOM mark is
+// prepended.
+//
+// Export for '_tkinter' shared extension
 PyAPI_FUNC(PyObject*) _PyUnicode_EncodeUTF16(
     PyObject* unicode,          /* Unicode object */
     const char *errors,         /* error handling */
@@ -297,13 +303,14 @@ extern PyObject* _PyUnicode_EncodeCharmap(
 
 /* --- Decimal Encoder ---------------------------------------------------- */
 
-/* Coverts a Unicode object holding a decimal value to an ASCII string
-   for using in int, float and complex parsers.
-   Transforms code points that have decimal digit property to the
-   corresponding ASCII digit code points.  Transforms spaces to ASCII.
-   Transforms code points starting from the first non-ASCII code point that
-   is neither a decimal digit nor a space to the end into '?'. */
-
+// Coverts a Unicode object holding a decimal value to an ASCII string
+// for using in int, float and complex parsers.
+// Transforms code points that have decimal digit property to the
+// corresponding ASCII digit code points.  Transforms spaces to ASCII.
+// Transforms code points starting from the first non-ASCII code point that
+// is neither a decimal digit nor a space to the end into '?'.
+//
+// Export for '_testinternalcapi' shared extension.
 PyAPI_FUNC(PyObject*) _PyUnicode_TransformDecimalAndSpaceToASCII(
     PyObject *unicode);         /* Unicode object */
 
@@ -323,9 +330,10 @@ extern int _PyUnicode_EqualToASCIIId(
     _Py_Identifier *right       /* Right identifier */
     );
 
-/* Test whether a unicode is equal to ASCII string.  Return 1 if true,
-   0 otherwise.  The right argument must be ASCII-encoded string.
-   Any error occurs inside will be cleared before return. */
+// Test whether a unicode is equal to ASCII string.  Return 1 if true,
+// 0 otherwise.  The right argument must be ASCII-encoded string.
+// Any error occurs inside will be cleared before return.
+// Export for '_ctypes' shared extension
 PyAPI_FUNC(int) _PyUnicode_EqualToASCIIString(
     PyObject *left,
     const char *right           /* ASCII-encoded string */
@@ -357,14 +365,17 @@ extern Py_ssize_t _PyUnicode_InsertThousandsGrouping(
 
 extern PyObject* _PyUnicode_FormatLong(PyObject *, int, int, int);
 
-/* Return an interned Unicode object for an Identifier; may fail if there is no memory.*/
+// Return an interned Unicode object for an Identifier; may fail if there is no
+// memory.
+// Export for '_testembed' program.
 PyAPI_FUNC(PyObject*) _PyUnicode_FromId(_Py_Identifier*);
 
 /* Fast equality check when the inputs are known to be exact unicode types
    and where the hash values are equal (i.e. a very probable match) */
 extern int _PyUnicode_EQ(PyObject *, PyObject *);
 
-/* Equality check. */
+// Equality check.
+// Export for '_pickle' shared extension.
 PyAPI_FUNC(int) _PyUnicode_Equal(PyObject *, PyObject *);
 
 extern int _PyUnicode_WideCharString_Converter(PyObject *, void *);