Use 21 bit characters on Lisp offering them.

XMLCONF

@@ -8,19 +8,19 @@ xmltest/not-wf/sa/007.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/008.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/009.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/010.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/011.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/011.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/012.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/012.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/013.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/013.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/014.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/014.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/015.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/015.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/016.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/016.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/017.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/018.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/019.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/020.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/020.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/021.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/021.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/022.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/022.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/023.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/023.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/024.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/025.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/026.xml [not validating:] not-wf [validating:] invalid
@@ -31,11 +31,11 @@ xmltest/not-wf/sa/030.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/031.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/032.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/033.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/034.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/034.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/035.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/036.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/037.xml [not validating:] not-wf [validating:] invalid
-xmltest/not-wf/sa/038.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/038.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/039.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/040.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/041.xml [not validating:] not-wf [validating:] invalid
@@ -73,7 +73,7 @@ xmltest/not-wf/sa/072.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/073.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/074.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/075.xml [not validating:] not-wf [validating:] not-wf
-xmltest/not-wf/sa/076.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/076.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/077.xml [not validating:] not-wf [validating:] not-wf
 xmltest/not-wf/sa/078.xml [not validating:] not-wf [validating:] not-wf
 xmltest/not-wf/sa/079.xml [not validating:] not-wf [validating:] not-wf
@@ -170,7 +170,7 @@ xmltest/not-wf/sa/169.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/170.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/171.xml [not validating:] not-wf [validating:] not-wf
 xmltest/not-wf/sa/172.xml [not validating:] not-wf [validating:] not-wf
-xmltest/not-wf/sa/173.xml [not validating:] not-wf [validating:] not-wf
+xmltest/not-wf/sa/173.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/174.xml [not validating:] not-wf [validating:] invalid
 xmltest/not-wf/sa/175.xml [not validating:] not-wf [validating:] not-wf
 xmltest/not-wf/sa/176.xml [not validating:] not-wf [validating:] not-wf
@@ -688,9 +688,9 @@ oasis/p03fail5.xml [not validating:] not-wf [validating:] not-wf
 oasis/p03fail7.xml [not validating:] not-wf [validating:] not-wf
 oasis/p03fail8.xml [not validating:] not-wf [validating:] not-wf
 oasis/p03fail9.xml [not validating:] not-wf [validating:] not-wf
-oasis/p04fail1.xml [not validating:] not-wf [validating:] not-wf
+oasis/p04fail1.xml [not validating:] not-wf [validating:] invalid
-oasis/p04fail2.xml [not validating:] not-wf [validating:] not-wf
+oasis/p04fail2.xml [not validating:] not-wf [validating:] invalid
-oasis/p04fail3.xml [not validating:] not-wf [validating:] not-wf
+oasis/p04fail3.xml [not validating:] not-wf [validating:] invalid
 oasis/p05fail1.xml [not validating:] not-wf [validating:] not-wf
 oasis/p05fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p05fail3.xml [not validating:] not-wf [validating:] not-wf
@@ -701,9 +701,9 @@ oasis/p09fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p09fail3.xml [not validating:] not-wf [validating:] not-wf
 oasis/p09fail4.xml [not validating:] not-wf [validating:] not-wf
 oasis/p09fail5.xml [not validating:] not-wf [validating:] not-wf
-oasis/p10fail1.xml [not validating:] not-wf [validating:] not-wf
+oasis/p10fail1.xml [not validating:] not-wf [validating:] invalid
-oasis/p10fail2.xml [not validating:] not-wf [validating:] not-wf
+oasis/p10fail2.xml [not validating:] not-wf [validating:] invalid
-oasis/p10fail3.xml [not validating:] not-wf [validating:] not-wf
+oasis/p10fail3.xml [not validating:] not-wf [validating:] invalid
 oasis/p11fail1.xml [not validating:] not-wf [validating:] not-wf
 oasis/p11fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p12fail1.xml [not validating:] not-wf [validating:] not-wf
@@ -752,24 +752,24 @@ oasis/p39fail2.xml [not validating:] not-wf [validating:] invalid
 oasis/p39fail3.xml [not validating:] not-wf [validating:] invalid
 oasis/p39fail4.xml [not validating:] not-wf [validating:] not-wf
 oasis/p39fail5.xml [not validating:] not-wf [validating:] not-wf
-oasis/p40fail1.xml [not validating:] not-wf [validating:] not-wf
+oasis/p40fail1.xml [not validating:] not-wf [validating:] invalid
 oasis/p40fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p40fail3.xml [not validating:] not-wf [validating:] not-wf
 oasis/p40fail4.xml [not validating:] not-wf [validating:] not-wf
 oasis/p41fail1.xml [not validating:] not-wf [validating:] not-wf
 oasis/p41fail2.xml [not validating:] not-wf [validating:] not-wf
-oasis/p41fail3.xml [not validating:] not-wf [validating:] not-wf
+oasis/p41fail3.xml [not validating:] not-wf [validating:] invalid
 oasis/p42fail1.xml [not validating:] not-wf [validating:] invalid
 oasis/p42fail2.xml [not validating:] not-wf [validating:] invalid
-oasis/p42fail3.xml [not validating:] not-wf [validating:] not-wf
+oasis/p42fail3.xml [not validating:] not-wf [validating:] invalid
 oasis/p43fail1.xml [not validating:] not-wf [validating:] not-wf
 oasis/p43fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p43fail3.xml [not validating:] not-wf [validating:] not-wf
 oasis/p44fail1.xml [not validating:] not-wf [validating:] not-wf
-oasis/p44fail2.xml [not validating:] not-wf [validating:] not-wf
+oasis/p44fail2.xml [not validating:] not-wf [validating:] invalid
-oasis/p44fail3.xml [not validating:] not-wf [validating:] not-wf
+oasis/p44fail3.xml [not validating:] not-wf [validating:] invalid
-oasis/p44fail4.xml [not validating:] not-wf [validating:] not-wf
+oasis/p44fail4.xml [not validating:] not-wf [validating:] invalid
-oasis/p44fail5.xml [not validating:] not-wf [validating:] not-wf
+oasis/p44fail5.xml [not validating:] not-wf [validating:] invalid
 oasis/p45fail1.xml [not validating:] not-wf [validating:] not-wf
 oasis/p45fail2.xml [not validating:] not-wf [validating:] not-wf
 oasis/p45fail3.xml [not validating:] not-wf [validating:] not-wf

doc/index.xml

@@ -35,28 +35,53 @@
     <p>
       The following libraries are available as separate downloads:
     </p>
-    <p>
+    <ul style="list-style-type: square">
-      &#x2b17;&#160;
+      <li>
-      <a href="http://www.lichteblau.com/cxml-rng/">cxml-rng</a>
+	<a href="http://www.lichteblau.com/cxml-rng/">cxml-rng</a>:
-      &#160;&#160;
+	Relax NG validation
-      Relax NG validation
+      </li>
-    </p>
+      <li>
-    <p>
+	<a href="http://www.lichteblau.com/cxml-stp/">cxml-stp</a>:
-      &#x2b17;&#160;
+	STP, an alternative to DOM
-      <a href="http://www.lichteblau.com/cxml-stp/">cxml-stp</a>
+      </li>
-      &#160;&#160;
+      <li>
-      STP, an alternative to DOM
+	<a href="http://common-lisp.net/project/plexippus-xpath/">Plexippus</a>:
-    </p>
+	XPath 1.0
-    <p>
+      </li>
-      &#x2b17;&#160;
+      <li>
-      <a href="http://common-lisp.net/project/closure/closure-html/">Closure
+	<a href="http://repo.or.cz/w/xuriella.git">Xuriella</a>:
-      HTML</a>: cxml can be used together with its sister project
+	XSLT 1.0 (work in progress)
-      Closure HTML to convert between HTML and XHTML.
+      </li>
-    </p>
+      <li>
+	<a href="http://common-lisp.net/project/closure/closure-html/">Closure
+	  HTML</a>:
+	parse HTML 4; convert between HTML and XHTML
+      </li>
+    </ul>
 
 
     <a name="changes"/>
     <h3>Recent Changes</h3>
+    <p class="nomargin"><tt>rel-2007-10-21</tt></p>
+    <ul class="nomargin">
+      <li>
+	Lisps using full 21 bit code points as characters are now fully
+	supported (including SBCL and Clozure CL) addition to the
+	existing support for 16 bit characters using UTF-16 (including
+	Allegro and LispWorks).  The feature <tt>rune-is-utf-16</tt> is
+	now set on 16 bit Lisps.<br/><br/> Make sure to update and
+	recompile both cxml and closure-common when upgrading.
+      </li>
+      <li>
+	New sink slot omit-xml-declaration-p; functions cxml:unescaped
+	and sax:unescaped.
+      </li>
+      <li>
+	Fixed various DTD serialization bugs.  Fixed xmls compatibility
+	bugs.  Fixed variable names in with-source.  Fixed
+	klacks-error export. 
+      </li>
+    </ul>
     <div style="background-color: #f7f7f7;
 		width: 60%;
 		border: solid #9c0000;

doc/installation.xml

@@ -24,12 +24,13 @@ cvs co closure-common</pre>
       supported by <a
       href="http://common-lisp.net/project/cl-plus-ssl/#trivial-gray-streams">trivial-gray-streams</a>.
     </p>
-    <ul>
+    <p>
-      <li>
+      On Lisps without Unicode support, cxml uses integers instead of
-	The SBCL port uses 16 bit surrogate characters instead of taking
+      characters to represent code points internally.  By default,
-	advantage of SBCL's full 21 bit character support.
+      however, the parser will recode all data to strings containing
-      </li>
+      characters representing UTF-8 bytes before passing it to user
-    </ul>
+      code.
+    </p>
 
     <a name="compilation"/>
     <h2>Compilation</h2>

xml/xml-parse.lisp

@@ -458,7 +458,8 @@
 
 (defmacro %put-unicode-char (code-var put)
   `(progn
-     (cond ((%> ,code-var #xFFFF)
+     (cond #+rune-is-utf-16
+           ((%> ,code-var #xFFFF)
           (,put (the rune (code-rune (%+ #xD7C0 (%ash ,code-var -10)))))
           (,put (the rune (code-rune (%ior #xDC00 (%and ,code-var #x03FF))))))
          (t
@@ -1489,19 +1490,14 @@
            value))))
 
 (definline data-rune-p (rune)
-  ;; any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
+  ;; Any Unicode character, excluding FFFE, and FFFF.
-  ;;
+  ;; Allow surrogates if using UTF-16, else allow >= 0x10000.
-  ;; FIXME: das halte ich fuer verkehrt.  Surrogates als Unicode-Zeichen
-  ;; sind verboten.  Das liegt hier aber nicht vor, denn wir arbeiten
-  ;; ja tatsaechlich mit UTF-16.  Verboten ist es nur, wenn wir ein
-  ;; solches Zeichen beim Dekodieren finden, das wird aber eben
-  ;; in encodings.lisp bereits geprueft.  --david
   (let ((c (rune-code rune)))
     (or (= c #x9) (= c #xA) (= c #xD)
         (<= #x20 c #xD7FF)
+	#+rune-is-utf-16 (<= #xD800 c #xDFFF)
         (<= #xE000 c #xFFFD)
-        (<= #xD800 c #xDBFF)
+	#-rune-is-utf-16 (<= #x10000 c #x10FFFF))))
-        (<= #xDC00 c #xDFFF))))
 
 (defun read-att-value (zinput input mode &optional canon-space-p (delim nil))
   (with-rune-collector-2 (collect)
@@ -1761,11 +1757,13 @@
       (rune= rune #/U+000D)))
 
 (defun code-data-char-p (c)
-  ;; any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
+  ;; Any Unicode character, excluding FFFE, and FFFF.
+  ;; Allow surrogates if using UTF-16, else allow >= 0x10000.
   (or (= c #x9) (= c #xA) (= c #xD)
       (<= #x20 c #xD7FF)
+      #+rune-is-utf-16 (<= #xD800 c #xDFFF)
       (<= #xE000 c #xFFFD)
-      (<= #x10000 c #x10FFFF)))
+      #-rune-is-utf-16 (<= #x10000 c #x10FFFF)))
 
 (defun pubid-char-p (c)
   (or (rune= c #/u+0020) (rune= c #/u+000D) (rune= c #/u+000A)