rfbSendFramebufferUpdate(rfbClientPtr cl,
                         sraRegionPtr givenUpdateRegion)
{
    sraRectangleIterator* i=NULL;
    sraRect rect;
    int nUpdateRegionRects;
    rfbFramebufferUpdateMsg *fu = (rfbFramebufferUpdateMsg *)cl->updateBuf;
    sraRegionPtr updateRegion,updateCopyRegion,tmpRegion;
    int dx, dy;
    rfbBool sendCursorShape = FALSE;
    rfbBool sendCursorPos = FALSE;
    rfbBool sendKeyboardLedState = FALSE;
    rfbBool sendSupportedMessages = FALSE;
    rfbBool sendSupportedEncodings = FALSE;
    rfbBool sendServerIdentity = FALSE;
    rfbBool result = TRUE;

    if(cl->screen->displayHook)
      cl->screen->displayHook(cl);

    /*
     * If framebuffer size was changed and the client supports NewFBSize
     * encoding, just send NewFBSize marker and return.
     */

    if (cl->useNewFBSize && cl->newFBSizePending) {
      LOCK(cl->updateMutex);
      cl->newFBSizePending = FALSE;
      UNLOCK(cl->updateMutex);
      fu->type = rfbFramebufferUpdate;
      fu->nRects = Swap16IfLE(1);
      cl->ublen = sz_rfbFramebufferUpdateMsg;

      if (cl->useExtDesktopSize) {
        if (!rfbSendExtDesktopSize(cl, cl->scaledScreen->width, cl->scaledScreen->height)) {
          if(cl->screen->displayFinishedHook)
            cl->screen->displayFinishedHook(cl, FALSE);
          return FALSE;
        }
      }
      else if (!rfbSendNewFBSize(cl, cl->scaledScreen->width, cl->scaledScreen->height)) {
	if(cl->screen->displayFinishedHook)
	  cl->screen->displayFinishedHook(cl, FALSE);
        return FALSE;
      }
      result = rfbSendUpdateBuf(cl);
      if(cl->screen->displayFinishedHook)
	cl->screen->displayFinishedHook(cl, result);
      return result;
    }

    /*
     * If this client understands cursor shape updates, cursor should be
     * removed from the framebuffer. Otherwise, make sure it's put up.
     */

    if (cl->enableCursorShapeUpdates) {
      if (cl->cursorWasChanged && cl->readyForSetColourMapEntries)
	  sendCursorShape = TRUE;
    }

    /*
     * Do we plan to send cursor position update?
     */

    if (cl->enableCursorPosUpdates && cl->cursorWasMoved)
      sendCursorPos = TRUE;

    /*
     * Do we plan to send a keyboard state update?
     */
    if ((cl->enableKeyboardLedState) &&
	(cl->screen->getKeyboardLedStateHook!=NULL))
    {
        int x;
        x=cl->screen->getKeyboardLedStateHook(cl->screen);
        if (x!=cl->lastKeyboardLedState)
        {
            sendKeyboardLedState = TRUE;
            cl->lastKeyboardLedState=x;
        }
    }

    /*
     * Do we plan to send a rfbEncodingSupportedMessages?
     */
    if (cl->enableSupportedMessages)
    {
        sendSupportedMessages = TRUE;
        /* We only send this message ONCE <per setEncodings message received>
         * (We disable it here)
         */
        cl->enableSupportedMessages = FALSE;
    }
    /*
     * Do we plan to send a rfbEncodingSupportedEncodings?
     */
    if (cl->enableSupportedEncodings)
    {
        sendSupportedEncodings = TRUE;
        /* We only send this message ONCE <per setEncodings message received>
         * (We disable it here)
         */
        cl->enableSupportedEncodings = FALSE;
    }
    /*
     * Do we plan to send a rfbEncodingServerIdentity?
     */
    if (cl->enableServerIdentity)
    {
        sendServerIdentity = TRUE;
        /* We only send this message ONCE <per setEncodings message received>
         * (We disable it here)
         */
        cl->enableServerIdentity = FALSE;
    }

    LOCK(cl->updateMutex);

    /*
     * The modifiedRegion may overlap the destination copyRegion.  We remove
     * any overlapping bits from the copyRegion (since they'd only be
     * overwritten anyway).
     */

    sraRgnSubtract(cl->copyRegion,cl->modifiedRegion);

    /*
     * The client is interested in the region requestedRegion.  The region
     * which should be updated now is the intersection of requestedRegion
     * and the union of modifiedRegion and copyRegion.  If it's empty then
     * no update is needed.
     */

    updateRegion = sraRgnCreateRgn(givenUpdateRegion);
    if(cl->screen->progressiveSliceHeight>0) {
	    int height=cl->screen->progressiveSliceHeight,
	    	y=cl->progressiveSliceY;
	    sraRegionPtr bbox=sraRgnBBox(updateRegion);
	    sraRect rect;
	    if(sraRgnPopRect(bbox,&rect,0)) {
		sraRegionPtr slice;
		if(y<rect.y1 || y>=rect.y2)
		    y=rect.y1;
	    	slice=sraRgnCreateRect(0,y,cl->screen->width,y+height);
		sraRgnAnd(updateRegion,slice);
		sraRgnDestroy(slice);
	    }
	    sraRgnDestroy(bbox);
	    y+=height;
	    if(y>=cl->screen->height)
		    y=0;
	    cl->progressiveSliceY=y;
    }

    sraRgnOr(updateRegion,cl->copyRegion);
    if(!sraRgnAnd(updateRegion,cl->requestedRegion) &&
       sraRgnEmpty(updateRegion) &&
       (cl->enableCursorShapeUpdates ||
	(cl->cursorX == cl->screen->cursorX && cl->cursorY == cl->screen->cursorY)) &&
       !sendCursorShape && !sendCursorPos && !sendKeyboardLedState &&
       !sendSupportedMessages && !sendSupportedEncodings && !sendServerIdentity) {
      sraRgnDestroy(updateRegion);
      UNLOCK(cl->updateMutex);
      if(cl->screen->displayFinishedHook)
	cl->screen->displayFinishedHook(cl, TRUE);
      return TRUE;
    }

    /*
     * We assume that the client doesn't have any pixel data outside the
     * requestedRegion.  In other words, both the source and destination of a
     * copy must lie within requestedRegion.  So the region we can send as a
     * copy is the intersection of the copyRegion with both the requestedRegion
     * and the requestedRegion translated by the amount of the copy.  We set
     * updateCopyRegion to this.
     */

    updateCopyRegion = sraRgnCreateRgn(cl->copyRegion);
    sraRgnAnd(updateCopyRegion,cl->requestedRegion);
    tmpRegion = sraRgnCreateRgn(cl->requestedRegion);
    sraRgnOffset(tmpRegion,cl->copyDX,cl->copyDY);
    sraRgnAnd(updateCopyRegion,tmpRegion);
    sraRgnDestroy(tmpRegion);
    dx = cl->copyDX;
    dy = cl->copyDY;

    /*
     * Next we remove updateCopyRegion from updateRegion so that updateRegion
     * is the part of this update which is sent as ordinary pixel data (i.e not
     * a copy).
     */

    sraRgnSubtract(updateRegion,updateCopyRegion);

    /*
     * Finally we leave modifiedRegion to be the remainder (if any) of parts of
     * the screen which are modified but outside the requestedRegion.  We also
     * empty both the requestedRegion and the copyRegion - note that we never
     * carry over a copyRegion for a future update.
     */

     sraRgnOr(cl->modifiedRegion,cl->copyRegion);
     sraRgnSubtract(cl->modifiedRegion,updateRegion);
     sraRgnSubtract(cl->modifiedRegion,updateCopyRegion);

     sraRgnMakeEmpty(cl->requestedRegion);
     sraRgnMakeEmpty(cl->copyRegion);
     cl->copyDX = 0;
     cl->copyDY = 0;

     UNLOCK(cl->updateMutex);

    if (!cl->enableCursorShapeUpdates) {
      if(cl->cursorX != cl->screen->cursorX || cl->cursorY != cl->screen->cursorY) {
	rfbRedrawAfterHideCursor(cl,updateRegion);
	LOCK(cl->screen->cursorMutex);
	cl->cursorX = cl->screen->cursorX;
	cl->cursorY = cl->screen->cursorY;
	UNLOCK(cl->screen->cursorMutex);
	rfbRedrawAfterHideCursor(cl,updateRegion);
      }
      rfbShowCursor(cl);
    }

    /*
     * Now send the update.
     */

    rfbStatRecordMessageSent(cl, rfbFramebufferUpdate, 0, 0);
    if (cl->preferredEncoding == rfbEncodingCoRRE) {
        nUpdateRegionRects = 0;

        for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
            int x = rect.x1;
            int y = rect.y1;
            int w = rect.x2 - x;
            int h = rect.y2 - y;
	    int rectsPerRow, rows;
            /* We need to count the number of rects in the scaled screen */
            if (cl->screen!=cl->scaledScreen)
                rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");
	    rectsPerRow = (w-1)/cl->correMaxWidth+1;
	    rows = (h-1)/cl->correMaxHeight+1;
	    nUpdateRegionRects += rectsPerRow*rows;
        }
	sraRgnReleaseIterator(i); i=NULL;
    } else if (cl->preferredEncoding == rfbEncodingUltra) {
        nUpdateRegionRects = 0;

        for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
            int x = rect.x1;
            int y = rect.y1;
            int w = rect.x2 - x;
            int h = rect.y2 - y;
            /* We need to count the number of rects in the scaled screen */
            if (cl->screen!=cl->scaledScreen)
                rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");
            nUpdateRegionRects += (((h-1) / (ULTRA_MAX_SIZE( w ) / w)) + 1);
          }
        sraRgnReleaseIterator(i); i=NULL;
#ifdef LIBVNCSERVER_HAVE_LIBZ
    } else if (cl->preferredEncoding == rfbEncodingZlib) {
	nUpdateRegionRects = 0;

        for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
            int x = rect.x1;
            int y = rect.y1;
            int w = rect.x2 - x;
            int h = rect.y2 - y;
            /* We need to count the number of rects in the scaled screen */
            if (cl->screen!=cl->scaledScreen)
                rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");
	    nUpdateRegionRects += (((h-1) / (ZLIB_MAX_SIZE( w ) / w)) + 1);
	}
	sraRgnReleaseIterator(i); i=NULL;
#ifdef LIBVNCSERVER_HAVE_LIBJPEG
    } else if (cl->preferredEncoding == rfbEncodingTight) {
	nUpdateRegionRects = 0;

        for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
            int x = rect.x1;
            int y = rect.y1;
            int w = rect.x2 - x;
            int h = rect.y2 - y;
            int n;
            /* We need to count the number of rects in the scaled screen */
            if (cl->screen!=cl->scaledScreen)
                rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");
	    n = rfbNumCodedRectsTight(cl, x, y, w, h);
	    if (n == 0) {
		nUpdateRegionRects = 0xFFFF;
		break;
	    }
	    nUpdateRegionRects += n;
	}
	sraRgnReleaseIterator(i); i=NULL;
#endif
#endif
#if defined(LIBVNCSERVER_HAVE_LIBJPEG) && defined(LIBVNCSERVER_HAVE_LIBPNG)
    } else if (cl->preferredEncoding == rfbEncodingTightPng) {
	nUpdateRegionRects = 0;

        for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
            int x = rect.x1;
            int y = rect.y1;
            int w = rect.x2 - x;
            int h = rect.y2 - y;
            int n;
            /* We need to count the number of rects in the scaled screen */
            if (cl->screen!=cl->scaledScreen)
                rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");
	    n = rfbNumCodedRectsTight(cl, x, y, w, h);
	    if (n == 0) {
		nUpdateRegionRects = 0xFFFF;
		break;
	    }
	    nUpdateRegionRects += n;
	}
	sraRgnReleaseIterator(i); i=NULL;
#endif
    } else {
        nUpdateRegionRects = sraRgnCountRects(updateRegion);
    }

    fu->type = rfbFramebufferUpdate;
    if (nUpdateRegionRects != 0xFFFF) {
	if(cl->screen->maxRectsPerUpdate>0
	   /* CoRRE splits the screen into smaller squares */
	   && cl->preferredEncoding != rfbEncodingCoRRE
	   /* Ultra encoding splits rectangles up into smaller chunks */
           && cl->preferredEncoding != rfbEncodingUltra
#ifdef LIBVNCSERVER_HAVE_LIBZ
	   /* Zlib encoding splits rectangles up into smaller chunks */
	   && cl->preferredEncoding != rfbEncodingZlib
#ifdef LIBVNCSERVER_HAVE_LIBJPEG
	   /* Tight encoding counts the rectangles differently */
	   && cl->preferredEncoding != rfbEncodingTight
#endif
#endif
#ifdef LIBVNCSERVER_HAVE_LIBPNG
	   /* Tight encoding counts the rectangles differently */
	   && cl->preferredEncoding != rfbEncodingTightPng
#endif
	   && nUpdateRegionRects>cl->screen->maxRectsPerUpdate) {
	    sraRegion* newUpdateRegion = sraRgnBBox(updateRegion);
	    sraRgnDestroy(updateRegion);
	    updateRegion = newUpdateRegion;
	    nUpdateRegionRects = sraRgnCountRects(updateRegion);
	}
	fu->nRects = Swap16IfLE((uint16_t)(sraRgnCountRects(updateCopyRegion) +
					   nUpdateRegionRects +
					   !!sendCursorShape + !!sendCursorPos + !!sendKeyboardLedState +
					   !!sendSupportedMessages + !!sendSupportedEncodings + !!sendServerIdentity));
    } else {
	fu->nRects = 0xFFFF;
    }
    cl->ublen = sz_rfbFramebufferUpdateMsg;

   if (sendCursorShape) {
	cl->cursorWasChanged = FALSE;
	if (!rfbSendCursorShape(cl))
	    goto updateFailed;
    }

   if (sendCursorPos) {
	cl->cursorWasMoved = FALSE;
	if (!rfbSendCursorPos(cl))
	        goto updateFailed;
   }

   if (sendKeyboardLedState) {
       if (!rfbSendKeyboardLedState(cl))
           goto updateFailed;
   }

   if (sendSupportedMessages) {
       if (!rfbSendSupportedMessages(cl))
           goto updateFailed;
   }
   if (sendSupportedEncodings) {
       if (!rfbSendSupportedEncodings(cl))
           goto updateFailed;
   }
   if (sendServerIdentity) {
       if (!rfbSendServerIdentity(cl))
           goto updateFailed;
   }

    if (!sraRgnEmpty(updateCopyRegion)) {
	if (!rfbSendCopyRegion(cl,updateCopyRegion,dx,dy))
	        goto updateFailed;
    }

    for(i = sraRgnGetIterator(updateRegion); sraRgnIteratorNext(i,&rect);){
        int x = rect.x1;
        int y = rect.y1;
        int w = rect.x2 - x;
        int h = rect.y2 - y;

        /* We need to count the number of rects in the scaled screen */
        if (cl->screen!=cl->scaledScreen)
            rfbScaledCorrection(cl->screen, cl->scaledScreen, &x, &y, &w, &h, "rfbSendFramebufferUpdate");

        switch (cl->preferredEncoding) {
	case -1:
        case rfbEncodingRaw:
            if (!rfbSendRectEncodingRaw(cl, x, y, w, h))
	        goto updateFailed;
            break;
        case rfbEncodingRRE:
            if (!rfbSendRectEncodingRRE(cl, x, y, w, h))
	        goto updateFailed;
            break;
        case rfbEncodingCoRRE:
            if (!rfbSendRectEncodingCoRRE(cl, x, y, w, h))
	        goto updateFailed;
	    break;
        case rfbEncodingHextile:
            if (!rfbSendRectEncodingHextile(cl, x, y, w, h))
	        goto updateFailed;
            break;
        case rfbEncodingUltra:
            if (!rfbSendRectEncodingUltra(cl, x, y, w, h))
                goto updateFailed;
            break;
#ifdef LIBVNCSERVER_HAVE_LIBZ
	case rfbEncodingZlib:
	    if (!rfbSendRectEncodingZlib(cl, x, y, w, h))
	        goto updateFailed;
	    break;
       case rfbEncodingZRLE:
       case rfbEncodingZYWRLE:
           if (!rfbSendRectEncodingZRLE(cl, x, y, w, h))
	       goto updateFailed;
           break;
#endif
#if defined(LIBVNCSERVER_HAVE_LIBJPEG) && (defined(LIBVNCSERVER_HAVE_LIBZ) || defined(LIBVNCSERVER_HAVE_LIBPNG))
	case rfbEncodingTight:
	    if (!rfbSendRectEncodingTight(cl, x, y, w, h))
	        goto updateFailed;
	    break;
#ifdef LIBVNCSERVER_HAVE_LIBPNG
	case rfbEncodingTightPng:
	    if (!rfbSendRectEncodingTightPng(cl, x, y, w, h))
	        goto updateFailed;
	    break;
#endif
#endif
        }
    }
    if (i) {
        sraRgnReleaseIterator(i);
        i = NULL;
    }

    if ( nUpdateRegionRects == 0xFFFF &&
	 !rfbSendLastRectMarker(cl) )
	    goto updateFailed;

    if (!rfbSendUpdateBuf(cl)) {
updateFailed:
	result = FALSE;
    }

    if (!cl->enableCursorShapeUpdates) {
      rfbHideCursor(cl);
    }

    if(i)
        sraRgnReleaseIterator(i);
    sraRgnDestroy(updateRegion);
    sraRgnDestroy(updateCopyRegion);

    if(cl->screen->displayFinishedHook)
      cl->screen->displayFinishedHook(cl, result);
    return result;
}


libvncserver/rfbserver: fix possible divide-by-zero

Closes #409

rfbNewTCPOrUDPClient(rfbScreenInfoPtr rfbScreen,
                     rfbSocket sock,
                     rfbBool isUDP)
{
    rfbProtocolVersionMsg pv;
    rfbClientIteratorPtr iterator;
    rfbClientPtr cl,cl_;
#ifdef LIBVNCSERVER_IPv6
    struct sockaddr_storage addr;
#else
    struct sockaddr_in addr;
#endif
    socklen_t addrlen = sizeof(addr);
    rfbProtocolExtension* extension;

    cl = (rfbClientPtr)calloc(sizeof(rfbClientRec),1);

    cl->screen = rfbScreen;
    cl->sock = sock;
    cl->viewOnly = FALSE;
    /* setup pseudo scaling */
    cl->scaledScreen = rfbScreen;
    cl->scaledScreen->scaledScreenRefCount++;

    rfbResetStats(cl);

    cl->clientData = NULL;
    cl->clientGoneHook = rfbDoNothingWithClient;

    if(isUDP) {
      rfbLog(" accepted UDP client\n");
	} else {
#ifdef LIBVNCSERVER_IPv6
		char host[1024];
#endif
      int one=1;
      size_t otherClientsCount = 0;

      getpeername(sock, (struct sockaddr *)&addr, &addrlen);
#ifdef LIBVNCSERVER_IPv6
      if(getnameinfo((struct sockaddr*)&addr, addrlen, host, sizeof(host), NULL, 0, NI_NUMERICHOST) != 0) {
	rfbLogPerror("rfbNewClient: error in getnameinfo");
	cl->host = strdup("");
      }
      else
	cl->host = strdup(host);
#else
      cl->host = strdup(inet_ntoa(addr.sin_addr));
#endif

      iterator = rfbGetClientIterator(rfbScreen);
      while ((cl_ = rfbClientIteratorNext(iterator)) != NULL)
	  ++otherClientsCount;
      rfbReleaseClientIterator(iterator);
      rfbLog("  %lu other clients\n", (unsigned long) otherClientsCount);

      if(!rfbSetNonBlocking(sock)) {
	rfbCloseSocket(sock);
	return NULL;
      }

      if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY,
		     (char *)&one, sizeof(one)) < 0) {
	rfbLogPerror("setsockopt failed: can't set TCP_NODELAY flag, non TCP socket?");
      }

      FD_SET(sock,&(rfbScreen->allFds));
		rfbScreen->maxFd = rfbMax(sock,rfbScreen->maxFd);

      INIT_MUTEX(cl->outputMutex);
      INIT_MUTEX(cl->refCountMutex);
      INIT_MUTEX(cl->sendMutex);
      INIT_COND(cl->deleteCond);

      cl->state = RFB_PROTOCOL_VERSION;

      cl->reverseConnection = FALSE;
      cl->readyForSetColourMapEntries = FALSE;
      cl->useCopyRect = FALSE;
      cl->preferredEncoding = -1;
      cl->correMaxWidth = 48;
      cl->correMaxHeight = 48;
#ifdef LIBVNCSERVER_HAVE_LIBZ
      cl->zrleData = NULL;
#endif

      cl->copyRegion = sraRgnCreate();
      cl->copyDX = 0;
      cl->copyDY = 0;

      cl->modifiedRegion =
	sraRgnCreateRect(0,0,rfbScreen->width,rfbScreen->height);

      INIT_MUTEX(cl->updateMutex);
      INIT_COND(cl->updateCond);

      cl->requestedRegion = sraRgnCreate();

      cl->format = cl->screen->serverFormat;
      cl->translateFn = rfbTranslateNone;
      cl->translateLookupTable = NULL;

      LOCK(rfbClientListMutex);
#if defined(LIBVNCSERVER_HAVE_LIBPTHREAD) || defined(LIBVNCSERVER_HAVE_WIN32THREADS)
      cl->refCount = 0;
#endif
      cl->next = rfbScreen->clientHead;
      cl->prev = NULL;
      if (rfbScreen->clientHead)
        rfbScreen->clientHead->prev = cl;

      rfbScreen->clientHead = cl;
      UNLOCK(rfbClientListMutex);

#if defined(LIBVNCSERVER_HAVE_LIBZ) || defined(LIBVNCSERVER_HAVE_LIBPNG)
      cl->tightQualityLevel = -1;
#ifdef LIBVNCSERVER_HAVE_LIBJPEG
      cl->tightCompressLevel = TIGHT_DEFAULT_COMPRESSION;
      cl->turboSubsampLevel = TURBO_DEFAULT_SUBSAMP;
      {
	int i;
	for (i = 0; i < 4; i++)
          cl->zsActive[i] = FALSE;
      }
#endif
#endif

      cl->fileTransfer.fd = -1;

      cl->enableCursorShapeUpdates = FALSE;
      cl->enableCursorPosUpdates = FALSE;
      cl->useRichCursorEncoding = FALSE;
      cl->enableLastRectEncoding = FALSE;
      cl->enableKeyboardLedState = FALSE;
      cl->enableSupportedMessages = FALSE;
      cl->enableSupportedEncodings = FALSE;
      cl->enableServerIdentity = FALSE;
      cl->lastKeyboardLedState = -1;
      cl->cursorX = rfbScreen->cursorX;
      cl->cursorY = rfbScreen->cursorY;
      cl->useNewFBSize = FALSE;
      cl->useExtDesktopSize = FALSE;
      cl->requestedDesktopSizeChange = 0;
      cl->lastDesktopSizeChangeError = 0;

#ifdef LIBVNCSERVER_HAVE_LIBZ
      cl->compStreamInited = FALSE;
      cl->compStream.total_in = 0;
      cl->compStream.total_out = 0;
      cl->compStream.zalloc = Z_NULL;
      cl->compStream.zfree = Z_NULL;
      cl->compStream.opaque = Z_NULL;

      cl->zlibCompressLevel = 5;
#endif

      cl->progressiveSliceY = 0;

      cl->extensions = NULL;

      cl->lastPtrX = -1;

#ifdef LIBVNCSERVER_HAVE_LIBPTHREAD
      cl->pipe_notify_client_thread[0] = -1;
      cl->pipe_notify_client_thread[1] = -1;
#endif

#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
      /*
       * Wait a few ms for the client to send WebSockets connection (TLS/SSL or plain)
       */
      if (!webSocketsCheck(cl)) {
        /* Error reporting handled in webSocketsHandshake */
        rfbCloseClient(cl);
        rfbClientConnectionGone(cl);
        return NULL;
      }
#endif

      sprintf(pv,rfbProtocolVersionFormat,rfbScreen->protocolMajorVersion,
              rfbScreen->protocolMinorVersion);

      if (rfbWriteExact(cl, pv, sz_rfbProtocolVersionMsg) < 0) {
        rfbLogPerror("rfbNewClient: write");
        rfbCloseClient(cl);
	rfbClientConnectionGone(cl);
        return NULL;
      }
    }

    for(extension = rfbGetExtensionIterator(); extension;
	    extension=extension->next) {
	void* data = NULL;
	/* if the extension does not have a newClient method, it wants
	 * to be initialized later. */
	if(extension->newClient && extension->newClient(cl, &data))
		rfbEnableExtension(cl, extension, data);
    }
    rfbReleaseExtensionIterator();

    switch (cl->screen->newClientHook(cl)) {
    case RFB_CLIENT_ON_HOLD:
	    cl->onHold = TRUE;
	    break;
    case RFB_CLIENT_ACCEPT:
	    cl->onHold = FALSE;
	    break;
    case RFB_CLIENT_REFUSE:
	    rfbCloseClient(cl);
	    rfbClientConnectionGone(cl);
	    cl = NULL;
	    break;
    }
    return cl;
}

A divide by zero issue was found to occur in libvncserver-0.9.12. A malicious client could use this flaw to send a specially crafted message that, when processed by the VNC server, would lead to a floating point exception, resulting in a denial of service.


rfbGetClientIteratorWithClosed(rfbScreenInfoPtr rfbScreen)
{
  rfbClientIteratorPtr i =
    (rfbClientIteratorPtr)malloc(sizeof(struct rfbClientIterator));
  i->next = NULL;
  i->screen = rfbScreen;
  i->closedToo = TRUE;
  return i;
}


rfbSendBell(rfbScreenInfoPtr rfbScreen)
{
    rfbClientIteratorPtr i;
    rfbClientPtr cl;
    rfbBellMsg b;

    i = rfbGetClientIterator(rfbScreen);
    while((cl=rfbClientIteratorNext(i))) {
	b.type = rfbBell;
        LOCK(cl->sendMutex);
	if (rfbWriteExact(cl, (char *)&b, sz_rfbBellMsg) < 0) {
	    rfbLogPerror("rfbSendBell: write");
	    rfbCloseClient(cl);
	}
        UNLOCK(cl->sendMutex);
    }
    rfbStatRecordMessageSent(cl, rfbBell, sz_rfbBellMsg, sz_rfbBellMsg);
    rfbReleaseClientIterator(i);
}


Fixed issue 22140 in oss-fuzz: Heap-buffer-overflow
* color quantized pix must be 8 bpp before extra colors are added.

pixQuantFromCmap(PIX      *pixs,
                 PIXCMAP  *cmap,
                 l_int32   mindepth,
                 l_int32   level,
                 l_int32   metric)
{
l_int32  d;

    PROCNAME("pixQuantFromCmap");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (mindepth != 2 && mindepth != 4 && mindepth != 8)
        return (PIX *)ERROR_PTR("invalid mindepth", procName, NULL);
    d = pixGetDepth(pixs);
    if (d == 8)
        return pixGrayQuantFromCmap(pixs, cmap, mindepth);
    else if (d == 32)
        return pixOctcubeQuantFromCmap(pixs, cmap, mindepth,
                                       level, metric);
    else
        return (PIX *)ERROR_PTR("d not 8 or 32 bpp", procName, NULL);
}

Leptonica before 1.80.0 allows a heap-based buffer over-read in pixFewColorsOctcubeQuantMixed in colorquant1.c.


octcubeGetCount(l_int32   level,
                l_int32  *psize)
{
    PROCNAME("octcubeGetCount");

    if (!psize)
        return ERROR_INT("&size not defined", procName, 1);
    if (level < 1 || level > 6)
        return ERROR_INT("invalid level", procName, 1);

    *psize = 1 << (3 * level);
    return 0;
}


pixOctreeQuantByPopulation(PIX     *pixs,
                           l_int32  level,
                           l_int32  ditherflag)
{
l_int32         w, h, wpls, wpld, i, j, depth, size, ncolors, index;
l_int32         rval, gval, bval;
l_int32        *rarray, *garray, *barray, *narray, *iarray;
l_uint32        octindex, octindex2;
l_uint32       *rtab, *gtab, *btab, *rtab2, *gtab2, *btab2;
l_uint32       *lines, *lined, *datas, *datad;
L_OCTCUBE_POP  *opop;
L_HEAP         *lh;
PIX            *pixd;
PIXCMAP        *cmap;

    PROCNAME("pixOctreeQuantByPopulation");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (level == 0) level = 4;
    if (level < 3 || level > 4)
        return (PIX *)ERROR_PTR("level not in {3,4}", procName, NULL);

        /* Do not dither if image is very small */
    pixGetDimensions(pixs, &w, &h, NULL);
    if (w < MinDitherSize && h < MinDitherSize && ditherflag == 1) {
        L_INFO("Small image: dithering turned off\n", procName);
        ditherflag = 0;
    }

    if (octcubeGetCount(level, &size))  /* array size = 2 ** (3 * level) */
        return (PIX *)ERROR_PTR("size not returned", procName, NULL);
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(level, &rtab, &gtab, &btab);

    pixd = NULL;
    narray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    rarray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    garray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    barray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    if (!narray || !rarray || !garray || !barray)
        goto array_cleanup;

        /* Place the pixels in octcube leaves. */
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j++) {
            extractRGBValues(lines[j], &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            narray[octindex]++;
            rarray[octindex] += rval;
            garray[octindex] += gval;
            barray[octindex] += bval;
        }
    }

        /* Find the number of different colors */
    for (i = 0, ncolors = 0; i < size; i++) {
        if (narray[i] > 0)
            ncolors++;
    }
    if (ncolors <= 4)
        depth = 2;
    else if (ncolors <= 16)
        depth = 4;
    else
        depth = 8;
    pixd = pixCreate(w, h, depth);
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    cmap = pixcmapCreate(depth);
    pixSetColormap(pixd, cmap);

        /* Average the colors in each octcube leaf. */
    for (i = 0; i < size; i++) {
        if (narray[i] > 0) {
            rarray[i] /= narray[i];
            garray[i] /= narray[i];
            barray[i] /= narray[i];
        }
    }

        /* If ncolors <= 256, finish immediately.  Do not dither.
         * Re-use narray to hold the colormap index + 1  */
    if (ncolors <= 256) {
        for (i = 0, index = 0; i < size; i++) {
            if (narray[i] > 0) {
                pixcmapAddColor(cmap, rarray[i], garray[i], barray[i]);
                narray[i] = index + 1;  /* to avoid storing 0 */
                index++;
            }
        }

            /* Set the cmap indices for each pixel */
        for (i = 0; i < h; i++) {
            lines = datas + i * wpls;
            lined = datad + i * wpld;
            for (j = 0; j < w; j++) {
                extractRGBValues(lines[j], &rval, &gval, &bval);
                octindex = rtab[rval] | gtab[gval] | btab[bval];
                switch (depth)
                {
                case 8:
                    SET_DATA_BYTE(lined, j, narray[octindex] - 1);
                    break;
                case 4:
                    SET_DATA_QBIT(lined, j, narray[octindex] - 1);
                    break;
                case 2:
                    SET_DATA_DIBIT(lined, j, narray[octindex] - 1);
                    break;
                default:
                    L_WARNING("shouldn't get here\n", procName);
                }
            }
        }
        goto array_cleanup;
    }

        /* More complicated.  Sort by decreasing population */
    lh = lheapCreate(500, L_SORT_DECREASING);
    for (i = 0; i < size; i++) {
        if (narray[i] > 0) {
            opop = (L_OCTCUBE_POP *)LEPT_CALLOC(1, sizeof(L_OCTCUBE_POP));
            opop->npix = (l_float32)narray[i];
            opop->index = i;
            opop->rval = rarray[i];
            opop->gval = garray[i];
            opop->bval = barray[i];
            lheapAdd(lh, opop);
        }
    }

        /* Take the top 192.  These will form the first 192 colors
         * in the cmap.  iarray[i] holds the index into the cmap. */
    iarray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    for (i = 0; i < 192; i++) {
        opop = (L_OCTCUBE_POP*)lheapRemove(lh);
        if (!opop) break;
        pixcmapAddColor(cmap, opop->rval, opop->gval, opop->bval);
        iarray[opop->index] = i + 1;  /* +1 to avoid storing 0 */

#if DEBUG_POP
        lept_stderr("i = %d, n = %6.0f, (r,g,b) = (%d %d %d)\n",
                    i, opop->npix, opop->rval, opop->gval, opop->bval);
#endif  /* DEBUG_POP */

        LEPT_FREE(opop);
    }

        /* Make the octindex tables for level 2, and reuse rarray, etc. */
    rtab2 = gtab2 = btab2 = NULL;
    makeRGBToIndexTables(2, &rtab2, &gtab2, &btab2);
    for (i = 0; i < 64; i++) {
        narray[i] = 0;
        rarray[i] = 0;
        garray[i] = 0;
        barray[i] = 0;
    }

        /* Take the rest of the occupied octcubes, assigning the pixels
         * to these new colormap indices.  iarray[] is addressed
         * by %level octcube indices, and it now holds the
         * colormap indices for all pixels in pixs.  */
    for (i = 192; i < size; i++) {
        opop = (L_OCTCUBE_POP*)lheapRemove(lh);
        if (!opop) break;
        rval = opop->rval;
        gval = opop->gval;
        bval = opop->bval;
        octindex2 = rtab2[rval] | gtab2[gval] | btab2[bval];
        narray[octindex2] += (l_int32)opop->npix;
        rarray[octindex2] += (l_int32)opop->npix * rval;
        garray[octindex2] += (l_int32)opop->npix * gval;
        barray[octindex2] += (l_int32)opop->npix * bval;
        iarray[opop->index] = 192 + octindex2 + 1;  /* +1 to avoid storing 0 */
        LEPT_FREE(opop);
    }
    lheapDestroy(&lh, TRUE);

        /* To span the full color space, which is necessary for dithering,
         * set each iarray element whose value is still 0 at the input
         * level octcube leaves (because there were no pixels in those
         * octcubes) to the colormap index corresponding to its level 2
         * octcube. */
    if (ditherflag) {
        for (i = 0; i < size; i++) {
            if (iarray[i] == 0) {
                getRGBFromOctcube(i, level, &rval, &gval, &bval);
                octindex2 = rtab2[rval] | gtab2[gval] | btab2[bval];
                iarray[i] = 192 + octindex2 + 1;
            }
        }
    }
    LEPT_FREE(rtab2);
    LEPT_FREE(gtab2);
    LEPT_FREE(btab2);

        /* Average the colors from the residuals in each level 2 octcube,
         * and add these 64 values to the colormap. */
    for (i = 0; i < 64; i++) {
        if (narray[i] > 0) {
            rarray[i] /= narray[i];
            garray[i] /= narray[i];
            barray[i] /= narray[i];
        } else {  /* no pixels in this octcube; use center value */
            getRGBFromOctcube(i, 2, &rarray[i], &garray[i], &barray[i]);
        }
        pixcmapAddColor(cmap, rarray[i], garray[i], barray[i]);
    }

        /* Set the cmap indices for each pixel.  Subtract 1 from
         * the value in iarray[] because we added 1 earlier.  */
    if (ditherflag == 0) {
        for (i = 0; i < h; i++) {
            lines = datas + i * wpls;
            lined = datad + i * wpld;
            for (j = 0; j < w; j++) {
                extractRGBValues(lines[j], &rval, &gval, &bval);
                octindex = rtab[rval] | gtab[gval] | btab[bval];
                SET_DATA_BYTE(lined, j, iarray[octindex] - 1);
            }
        }
    } else {  /* dither */
        pixDitherOctindexWithCmap(pixs, pixd, rtab, gtab, btab,
                                  iarray, POP_DIF_CAP);
    }

#if DEBUG_POP
    for (i = 0; i < size / 16; i++) {
        l_int32 j;
        for (j = 0; j < 16; j++)
            lept_stderr("%d ", iarray[16 * i + j]);
        lept_stderr("\n");
    }
#endif  /* DEBUG_POP */

    LEPT_FREE(iarray);

array_cleanup:
    LEPT_FREE(narray);
    LEPT_FREE(rarray);
    LEPT_FREE(garray);
    LEPT_FREE(barray);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);

    return pixd;
}


octreeGenerateAndPrune(PIX       *pixs,
                       l_int32    colors,
                       l_int32    reservedcolors,
                       PIXCMAP  **pcmap)
{
l_int32    rval, gval, bval, cindex;
l_int32    level, ncells, octindex;
l_int32    w, h, wpls;
l_int32    i, j, isub;
l_int32    npix;  /* number of remaining pixels to be assigned */
l_int32    ncolor; /* number of remaining color cells to be used */
l_int32    ppc;  /* ave number of pixels left for each color cell */
l_int32    rv, gv, bv;
l_float32  thresholdFactor[] = {0.01f, 0.01f, 1.0f, 1.0f, 1.0f, 1.0f};
l_float32  thresh;  /* factor of ppc for this level */
l_uint32  *datas, *lines;
l_uint32  *rtab, *gtab, *btab;
CQCELL  ***cqcaa;   /* one array for each octree level */
CQCELL   **cqca, **cqcasub;
CQCELL    *cqc, *cqcsub;
PIXCMAP   *cmap;
NUMA      *nat;  /* accumulates levels for threshold cells */
NUMA      *nar;  /* accumulates levels for residual cells */

    PROCNAME("octreeGenerateAndPrune");

    if (!pixs)
        return (CQCELL ***)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (CQCELL ***)ERROR_PTR("pixs must be 32 bpp", procName, NULL);
    if (colors < 128 || colors > 256)
        return (CQCELL ***)ERROR_PTR("colors not in [128,256]", procName, NULL);
    if (!pcmap)
        return (CQCELL ***)ERROR_PTR("&cmap not defined", procName, NULL);

    if ((cqcaa = cqcellTreeCreate()) == NULL)
        return (CQCELL ***)ERROR_PTR("cqcaa not made", procName, NULL);

        /* Make the canonical index tables */
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(CqNLevels, &rtab, &gtab, &btab);

        /* Generate an 8 bpp cmap (max size 256) */
    cmap = pixcmapCreate(8);
    *pcmap = cmap;

    pixGetDimensions(pixs, &w, &h, NULL);
    npix = w * h;  /* initialize to all pixels */
    ncolor = colors - reservedcolors - ExtraReservedColors;
    ppc = npix / ncolor;
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);

        /* Accumulate the centers of each cluster at level CqNLevels */
    ncells = 1 << (3 * CqNLevels);
    cqca = cqcaa[CqNLevels];
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j++) {
            extractRGBValues(lines[j], &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            cqc = cqca[octindex];
            cqc->n++;
        }
    }

        /* Arrays for storing statistics */
    nat = numaCreate(0);
    nar = numaCreate(0);

        /* Prune back from the lowest level and generate the colormap */
    for (level = CqNLevels - 1; level >= 2; level--) {
        thresh = thresholdFactor[level];
        cqca = cqcaa[level];
        cqcasub = cqcaa[level + 1];
        ncells = 1 << (3 * level);
        for (i = 0; i < ncells; i++) {  /* i is octindex at level */
            cqc = cqca[i];
            for (j = 0; j < 8; j++) {  /* check all subnodes */
                isub = 8 * i + j;   /* isub is octindex at level+1 */
                cqcsub = cqcasub[isub];
                if (cqcsub->bleaf == 1) {  /* already a leaf? */
                    cqc->nleaves++;   /* count the subcube leaves */
                    continue;
                }
                if (cqcsub->n >= thresh * ppc) {  /* make it a true leaf? */
                    cqcsub->bleaf = 1;
                    if (cmap->n < 256) {
                        cqcsub->index = cmap->n;  /* assign the color index */
                        getRGBFromOctcube(isub, level + 1, &rv, &gv, &bv);
                        pixcmapAddColor(cmap, rv, gv, bv);
#if 1   /* save values */
                        cqcsub->rc = rv;
                        cqcsub->gc = gv;
                        cqcsub->bc = bv;
#endif
                    } else {
                            /* This doesn't seem to happen. Do something. */
                        L_ERROR("assigning pixels to wrong color\n", procName);
                        pixcmapGetNearestIndex(cmap, 128, 128, 128, &cindex);
                        cqcsub->index = cindex;  /* assign to the nearest */
                        pixcmapGetColor(cmap, cindex, &rval, &gval, &bval);
                        cqcsub->rc = rval;
                        cqcsub->gc = gval;
                        cqcsub->bc = bval;
                    }
                    cqc->nleaves++;
                    npix -= cqcsub->n;
                    ncolor--;
                    if (ncolor > 0)
                        ppc = npix / ncolor;
                    else if (ncolor + reservedcolors > 0)
                        ppc = npix / (ncolor + reservedcolors);
                    else
                        ppc = 1000000;  /* make it big */
                    numaAddNumber(nat, level + 1);

#if  DEBUG_OCTCUBE_CMAP
    lept_stderr("Exceeds threshold: colors used = %d, colors remaining = %d\n",
                cmap->n, ncolor + reservedcolors);
    lept_stderr("  cell with %d pixels, npix = %d, ppc = %d\n",
                cqcsub->n, npix, ppc);
    lept_stderr("  index = %d, level = %d, subindex = %d\n",
                i, level, j);
    lept_stderr("  rv = %d, gv = %d, bv = %d\n", rv, gv, bv);
#endif  /* DEBUG_OCTCUBE_CMAP */

                }
            }
            if (cqc->nleaves > 0 || level == 2) { /* make the cube a leaf now */
                cqc->bleaf = 1;
                if (cqc->nleaves < 8) {  /* residual CTE cube: acquire the
                                          * remaining pixels */
                    for (j = 0; j < 8; j++) {  /* check all subnodes */
                        isub = 8 * i + j;
                        cqcsub = cqcasub[isub];
                        if (cqcsub->bleaf == 0)  /* absorb */
                            cqc->n += cqcsub->n;
                    }
                    if (cmap->n < 256) {
                        cqc->index = cmap->n;  /* assign the color index */
                        getRGBFromOctcube(i, level, &rv, &gv, &bv);
                        pixcmapAddColor(cmap, rv, gv, bv);
#if 1   /* save values */
                        cqc->rc = rv;
                        cqc->gc = gv;
                        cqc->bc = bv;
#endif
                    } else {
                        L_WARNING("possibly assigned pixels to wrong color\n",
                                  procName);
                            /* This is very bad.  It will only cause trouble
                             * with dithering, and we try to avoid it with
                             * ExtraReservedColors. */
                        pixcmapGetNearestIndex(cmap, rv, gv, bv, &cindex);
                        cqc->index = cindex;  /* assign to the nearest */
                        pixcmapGetColor(cmap, cindex, &rval, &gval, &bval);
                        cqc->rc = rval;
                        cqc->gc = gval;
                        cqc->bc = bval;
                    }
                    npix -= cqc->n;
                    ncolor--;
                    if (ncolor > 0)
                        ppc = npix / ncolor;
                    else if (ncolor + reservedcolors > 0)
                        ppc = npix / (ncolor + reservedcolors);
                    else
                        ppc = 1000000;  /* make it big */
                    numaAddNumber(nar, level);

#if  DEBUG_OCTCUBE_CMAP
    lept_stderr("By remainder: colors used = %d, colors remaining = %d\n",
                cmap->n, ncolor + reservedcolors);
    lept_stderr("  cell with %d pixels, npix = %d, ppc = %d\n",
                cqc->n, npix, ppc);
    lept_stderr("  index = %d, level = %d\n", i, level);
    lept_stderr("  rv = %d, gv = %d, bv = %d\n", rv, gv, bv);
#endif  /* DEBUG_OCTCUBE_CMAP */

                }
            } else {  /* absorb all the subpixels but don't make it a leaf */
                for (j = 0; j < 8; j++) {  /* absorb from all subnodes */
                    isub = 8 * i + j;
                    cqcsub = cqcasub[isub];
                    cqc->n += cqcsub->n;
                }
            }
        }
    }

#if  PRINT_OCTCUBE_STATS
{
l_int32    tc[] = {0, 0, 0, 0, 0, 0, 0};
l_int32    rc[] = {0, 0, 0, 0, 0, 0, 0};
l_int32    nt, nr, ival;

    nt = numaGetCount(nat);
    nr = numaGetCount(nar);
    for (i = 0; i < nt; i++) {
        numaGetIValue(nat, i, &ival);
        tc[ival]++;
    }
    for (i = 0; i < nr; i++) {
        numaGetIValue(nar, i, &ival);
        rc[ival]++;
    }
    lept_stderr(" Threshold cells formed: %d\n", nt);
    for (i = 1; i < CqNLevels + 1; i++)
        lept_stderr("   level %d:  %d\n", i, tc[i]);
    lept_stderr("\n Residual cells formed: %d\n", nr);
    for (i = 0; i < CqNLevels ; i++)
        lept_stderr("   level %d:  %d\n", i, rc[i]);
}
#endif  /* PRINT_OCTCUBE_STATS */

    numaDestroy(&nat);
    numaDestroy(&nar);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);

    return cqcaa;
}


pixOctreeQuantNumColors(PIX     *pixs,
                        l_int32  maxcolors,
                        l_int32  subsample)
{
l_int32    w, h, minside, bpp, wpls, wpld, i, j, actualcolors;
l_int32    rval, gval, bval, nbase, nextra, maxlevel, ncubes, val;
l_int32   *lut1, *lut2;
l_uint32   index;
l_uint32  *lines, *lined, *datas, *datad, *pspixel;
l_uint32  *rtab, *gtab, *btab;
OQCELL    *oqc;
OQCELL   **oqca;
L_HEAP    *lh;
PIX       *pixd;
PIXCMAP   *cmap;

    PROCNAME("pixOctreeQuantNumColors");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (maxcolors < 8) {
        L_WARNING("max colors < 8; setting to 8\n", procName);
        maxcolors = 8;
    }
    if (maxcolors > 256) {
        L_WARNING("max colors > 256; setting to 256\n", procName);
        maxcolors = 256;
    }

    pixGetDimensions(pixs, &w, &h, NULL);
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    minside = L_MIN(w, h);
    if (subsample <= 0) {
       subsample = L_MAX(1, minside / 200);
    }

    if (maxcolors <= 16) {
        bpp = 4;
        pixd = pixCreate(w, h, bpp);
        maxlevel = 2;
        ncubes = 64;   /* 2^6 */
        nbase = 8;
        nextra = maxcolors - nbase;
    } else if (maxcolors <= 64) {
        bpp = 8;
        pixd = pixCreate(w, h, bpp);
        maxlevel = 2;
        ncubes = 64;  /* 2^6 */
        nbase = 8;
        nextra = maxcolors - nbase;
    } else {  /* maxcolors <= 256 */
        bpp = 8;
        pixd = pixCreate(w, h, bpp);
        maxlevel = 3;
        ncubes = 512;  /* 2^9 */
        nbase = 64;
        nextra = maxcolors - nbase;
    }

    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);

        /*----------------------------------------------------------*
         * If we're using the minimum number of colors, it is       *
         * much simpler.  We just use 'nbase' octcubes.             *
         * For this case, we don't eliminate any extra colors.      *
         *----------------------------------------------------------*/
    if (nextra == 0) {
            /* prepare the OctcubeQuantCell array */
        if ((oqca = (OQCELL **)LEPT_CALLOC(nbase, sizeof(OQCELL *))) == NULL) {
            pixDestroy(&pixd);
            return (PIX *)ERROR_PTR("oqca not made", procName, NULL);
        }
        for (i = 0; i < nbase; i++) {
            oqca[i] = (OQCELL *)LEPT_CALLOC(1, sizeof(OQCELL));
            oqca[i]->n = 0.0;
        }

        rtab = gtab = btab = NULL;
        makeRGBToIndexTables(maxlevel - 1, &rtab, &gtab, &btab);

            /* Go through the entire image, gathering statistics and
             * assigning pixels to their quantized value */
        datad = pixGetData(pixd);
        wpld = pixGetWpl(pixd);
        for (i = 0; i < h; i++) {
            lines = datas + i * wpls;
            lined = datad + i * wpld;
            for (j = 0; j < w; j++) {
                pspixel = lines + j;
                extractRGBValues(*pspixel, &rval, &gval, &bval);
                getOctcubeIndexFromRGB(rval, gval, bval,
                                       rtab, gtab, btab, &index);
/*                lept_stderr("rval = %d, gval = %d, bval = %d,"
                              " index = %d\n", rval, gval, bval, index); */
                if (bpp == 4)
                    SET_DATA_QBIT(lined, j, index);
                else  /* bpp == 8 */
                    SET_DATA_BYTE(lined, j, index);
                oqca[index]->n += 1.0;
                oqca[index]->rcum += rval;
                oqca[index]->gcum += gval;
                oqca[index]->bcum += bval;
            }
        }

            /* Compute average color values in each octcube, and
             * generate colormap */
        cmap = pixcmapCreate(bpp);
        pixSetColormap(pixd, cmap);
        for (i = 0; i < nbase; i++) {
            oqc = oqca[i];
            if (oqc->n != 0) {
                oqc->rval = (l_int32)(oqc->rcum / oqc->n);
                oqc->gval = (l_int32)(oqc->gcum / oqc->n);
                oqc->bval = (l_int32)(oqc->bcum / oqc->n);
            } else {
                getRGBFromOctcube(i, maxlevel - 1, &oqc->rval,
                                  &oqc->gval, &oqc->bval);
            }
            pixcmapAddColor(cmap, oqc->rval, oqc->gval, oqc->bval);
        }

        for (i = 0; i < nbase; i++)
            LEPT_FREE(oqca[i]);
        LEPT_FREE(oqca);
        LEPT_FREE(rtab);
        LEPT_FREE(gtab);
        LEPT_FREE(btab);
        return pixd;
    }

        /*------------------------------------------------------------*
         * General case: we will use colors in octcubes at maxlevel.  *
         * We also remove any colors that are not populated from      *
         * the colormap.                                              *
         *------------------------------------------------------------*/
        /* Prepare the OctcubeQuantCell array */
    if ((oqca = (OQCELL **)LEPT_CALLOC(ncubes, sizeof(OQCELL *))) == NULL) {
        pixDestroy(&pixd);
        return (PIX *)ERROR_PTR("oqca not made", procName, NULL);
    }
    for (i = 0; i < ncubes; i++) {
        oqca[i] = (OQCELL *)LEPT_CALLOC(1, sizeof(OQCELL));
        oqca[i]->n = 0.0;
    }

        /* Make the tables to map color to the octindex,
         * of which there are 'ncubes' at 'maxlevel' */
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(maxlevel, &rtab, &gtab, &btab);

        /* Estimate the color distribution; we want to find the
         * most popular nextra colors at 'maxlevel' */
    for (i = 0; i < h; i += subsample) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j += subsample) {
            pspixel = lines + j;
            extractRGBValues(*pspixel, &rval, &gval, &bval);
            getOctcubeIndexFromRGB(rval, gval, bval, rtab, gtab, btab, &index);
            oqca[index]->n += 1.0;
            oqca[index]->octindex = index;
            oqca[index]->rcum += rval;
            oqca[index]->gcum += gval;
            oqca[index]->bcum += bval;
        }
    }

        /* Transfer the OQCELL from the array, and order in a heap */
    lh = lheapCreate(512, L_SORT_DECREASING);
    for (i = 0; i < ncubes; i++)
        lheapAdd(lh, oqca[i]);
    LEPT_FREE(oqca);  /* don't need this array */

        /* Prepare a new OctcubeQuantCell array, with maxcolors cells  */
    oqca = (OQCELL **)LEPT_CALLOC(maxcolors, sizeof(OQCELL *));
    for (i = 0; i < nbase; i++) {  /* make nbase cells */
        oqca[i] = (OQCELL *)LEPT_CALLOC(1, sizeof(OQCELL));
        oqca[i]->n = 0.0;
    }

        /* Remove the nextra most populated ones, and put them in the array */
    for (i = 0; i < nextra; i++) {
        oqc = (OQCELL *)lheapRemove(lh);
        oqc->n = 0.0;  /* reinit */
        oqc->rcum = 0;
        oqc->gcum = 0;
        oqc->bcum = 0;
        oqca[nbase + i] = oqc;  /* store it in the array */
    }

        /* Destroy the heap and its remaining contents */
    lheapDestroy(&lh, TRUE);

        /* Generate a lookup table from octindex at maxlevel
         * to color table index */
    lut1 = (l_int32 *)LEPT_CALLOC(ncubes, sizeof(l_int32));
    for (i = 0; i < nextra; i++)
        lut1[oqca[nbase + i]->octindex] = nbase + i;
    for (index = 0; index < ncubes; index++) {
        if (lut1[index] == 0)  /* not one of the extras; need to assign */
            lut1[index] = index >> 3;  /* remove the least significant bits */
/*        lept_stderr("lut1[%d] = %d\n", index, lut1[index]); */
    }

        /* Go through the entire image, gathering statistics and
         * assigning pixels to their quantized value */
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            pspixel = lines + j;
            extractRGBValues(*pspixel, &rval, &gval, &bval);
            getOctcubeIndexFromRGB(rval, gval, bval, rtab, gtab, btab, &index);
/*            lept_stderr("rval = %d, gval = %d, bval = %d, index = %d\n",
                          rval, gval, bval, index); */
            val = lut1[index];
            switch (bpp) {
            case 4:
                SET_DATA_QBIT(lined, j, val);
                break;
            case 8:
                SET_DATA_BYTE(lined, j, val);
                break;
            default:
                LEPT_FREE(oqca);
                LEPT_FREE(lut1);
                return (PIX *)ERROR_PTR("bpp not 4 or 8!", procName, NULL);
                break;
            }
            oqca[val]->n += 1.0;
            oqca[val]->rcum += rval;
            oqca[val]->gcum += gval;
            oqca[val]->bcum += bval;
        }
    }

        /* Compute averages, set up a colormap, and make a second
         * lut that converts from the color values currently in
         * the image to a minimal set */
    lut2 = (l_int32 *)LEPT_CALLOC(ncubes, sizeof(l_int32));
    cmap = pixcmapCreate(bpp);
    pixSetColormap(pixd, cmap);
    for (i = 0, index = 0; i < maxcolors; i++) {
        oqc = oqca[i];
        lut2[i] = index;
        if (oqc->n == 0)  /* no occupancy; don't bump up index */
            continue;
        oqc->rval = (l_int32)(oqc->rcum / oqc->n);
        oqc->gval = (l_int32)(oqc->gcum / oqc->n);
        oqc->bval = (l_int32)(oqc->bcum / oqc->n);
        pixcmapAddColor(cmap, oqc->rval, oqc->gval, oqc->bval);
        index++;
    }
/*    pixcmapWriteStream(stderr, cmap); */
    actualcolors = pixcmapGetCount(cmap);
/*    lept_stderr("Number of different colors = %d\n", actualcolors); */

        /* Last time through the image; use the lookup table to
         * remap the pixel value to the minimal colormap */
    if (actualcolors < maxcolors) {
        for (i = 0; i < h; i++) {
            lined = datad + i * wpld;
            for (j = 0; j < w; j++) {
                switch (bpp) {
                case 4:
                    val = GET_DATA_QBIT(lined, j);
                    SET_DATA_QBIT(lined, j, lut2[val]);
                    break;
                case 8:
                    val = GET_DATA_BYTE(lined, j);
                    SET_DATA_BYTE(lined, j, lut2[val]);
                    break;
                }
            }
        }
    }

    if (oqca) {
        for (i = 0; i < maxcolors; i++)
            LEPT_FREE(oqca[i]);
    }
    LEPT_FREE(oqca);
    LEPT_FREE(lut1);
    LEPT_FREE(lut2);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    return pixd;
}


cqcellTreeCreate(void)
{
l_int32    level, ncells, i;
CQCELL  ***cqcaa;
CQCELL   **cqca;   /* one array for each octree level */

    PROCNAME("cqcellTreeCreate");

        /* Make array of accumulation cell arrays from levels 1 to 5 */
    if ((cqcaa = (CQCELL ***)LEPT_CALLOC(CqNLevels + 1, sizeof(CQCELL **)))
        == NULL)
        return (CQCELL ***)ERROR_PTR("cqcaa not made", procName, NULL);
    for (level = 0; level <= CqNLevels; level++) {
        ncells = 1 << (3 * level);
        if ((cqca = (CQCELL **)LEPT_CALLOC(ncells, sizeof(CQCELL *))) == NULL) {
            cqcellTreeDestroy(&cqcaa);
            return (CQCELL ***)ERROR_PTR("cqca not made", procName, NULL);
        }
        cqcaa[level] = cqca;
        for (i = 0; i < ncells; i++) {
            if ((cqca[i] = (CQCELL *)LEPT_CALLOC(1, sizeof(CQCELL))) == NULL) {
                cqcellTreeDestroy(&cqcaa);
                return (CQCELL ***)ERROR_PTR("cqc not made", procName, NULL);
            }
        }
    }

    return cqcaa;
}


pixFixedOctcubeQuant256(PIX     *pixs,
                        l_int32  ditherflag)
{
l_uint8    index;
l_int32    rval, gval, bval;
l_int32    w, h, wpls, wpld, i, j, cindex;
l_uint32  *rtab, *gtab, *btab;
l_int32   *itab;
l_uint32  *datas, *datad, *lines, *lined;
PIX       *pixd;
PIXCMAP   *cmap;

    PROCNAME("pixFixedOctcubeQuant256");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);

        /* Do not dither if image is very small */
    pixGetDimensions(pixs, &w, &h, NULL);
    if (w < MinDitherSize && h < MinDitherSize && ditherflag == 1) {
        L_INFO("Small image: dithering turned off\n", procName);
        ditherflag = 0;
    }

        /* Find the centers of the 256 cells, each of which represents
         * the 3 MSBits of the red and green components, and the
         * 2 MSBits of the blue component.  This gives a mapping
         * from a "cube index" to the rgb values.  Save all 256
         * rgb values of these centers in a colormap.
         * For example, to get the red color of the cell center,
         * you take the 3 MSBits of to the index and add the
         * offset to the center of the cell, which is 0x10. */
    cmap = pixcmapCreate(8);
    for (cindex = 0; cindex < 256; cindex++) {
        rval = (cindex & 0xe0) | 0x10;
        gval = ((cindex << 3) & 0xe0) | 0x10;
        bval = ((cindex << 6) & 0xc0) | 0x20;
        pixcmapAddColor(cmap, rval, gval, bval);
    }

        /* Make output 8 bpp palette image */
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    if ((pixd = pixCreate(w, h, 8)) == NULL) {
        pixcmapDestroy(&cmap);
        return (PIX *)ERROR_PTR("pixd not made", procName, NULL);
    }
    pixSetColormap(pixd, cmap);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);

        /* Set dest pix values to colortable indices */
    if (ditherflag == 0) {   /* no dithering */
        for (i = 0; i < h; i++) {
            lines = datas + i * wpls;
            lined = datad + i * wpld;
            for (j = 0; j < w; j++) {
                extractRGBValues(lines[j], &rval, &gval, &bval);
                index = (rval & 0xe0) | ((gval >> 3) & 0x1c) | (bval >> 6);
                SET_DATA_BYTE(lined, j, index);
            }
        }
    } else {  /* ditherflag == 1 */
            /* Set up conversion tables from rgb directly to the colormap
             * index.  However, the dithering function expects these tables
             * to generate an octcube index (+1), and the table itab[] to
             * convert to the colormap index.  So we make a trivial
             * itab[], that simply compensates for the -1 in
             * pixDitherOctindexWithCmap().   No cap is required on
             * the propagated difference.  */
        rtab = (l_uint32 *)LEPT_CALLOC(256, sizeof(l_uint32));
        gtab = (l_uint32 *)LEPT_CALLOC(256, sizeof(l_uint32));
        btab = (l_uint32 *)LEPT_CALLOC(256, sizeof(l_uint32));
        itab = (l_int32 *)LEPT_CALLOC(256, sizeof(l_int32));
        if (!rtab || !gtab || !btab || !itab) {
            pixDestroy(&pixd);
            return (PIX *)ERROR_PTR("calloc fail for table", procName, NULL);
        }
        for (i = 0; i < 256; i++) {
            rtab[i] = i & 0xe0;
            gtab[i] = (i >> 3) & 0x1c;
            btab[i] = i >> 6;
            itab[i] = i + 1;
        }
        pixDitherOctindexWithCmap(pixs, pixd, rtab, gtab, btab, itab,
                                  FIXED_DIF_CAP);
        LEPT_FREE(rtab);
        LEPT_FREE(gtab);
        LEPT_FREE(btab);
        LEPT_FREE(itab);
    }

    return pixd;
}


pixFewColorsOctcubeQuant1(PIX     *pixs,
                          l_int32  level)
{
l_int32    w, h, wpls, wpld, i, j, depth, size, ncolors, index;
l_int32    rval, gval, bval;
l_int32   *carray, *rarray, *garray, *barray;
l_uint32   octindex;
l_uint32  *rtab, *gtab, *btab;
l_uint32  *lines, *lined, *datas, *datad, *pspixel;
PIX       *pixd;
PIXCMAP   *cmap;

    PROCNAME("pixFewColorsOctcubeQuant1");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (level < 1 || level > 6)
        return (PIX *)ERROR_PTR("invalid level", procName, NULL);

    pixd = NULL;

    if (octcubeGetCount(level, &size))  /* array size = 2 ** (3 * level) */
        return (PIX *)ERROR_PTR("size not returned", procName, NULL);
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(level, &rtab, &gtab, &btab);

    carray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    rarray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    garray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    barray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    if (!carray || !rarray || !garray || !barray) {
        L_ERROR("calloc fail for an array\n", procName);
        goto array_cleanup;
    }

        /* Place the pixels in octcube leaves. */
    pixGetDimensions(pixs, &w, &h, NULL);
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j++) {
            pspixel = lines + j;
            extractRGBValues(*pspixel, &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            carray[octindex]++;
            rarray[octindex] += rval;
            garray[octindex] += gval;
            barray[octindex] += bval;
        }
    }

        /* Find the number of different colors */
    for (i = 0, ncolors = 0; i < size; i++) {
        if (carray[i] > 0)
            ncolors++;
    }
    if (ncolors > 256) {
        L_WARNING("%d colors found; more than 256\n", procName, ncolors);
        goto array_cleanup;
    }
    if (ncolors <= 4)
        depth = 2;
    else if (ncolors <= 16)
        depth = 4;
    else
        depth = 8;

        /* Average the colors in each octcube leaf and add to colormap table;
         * then use carray to hold the colormap index + 1  */
    cmap = pixcmapCreate(depth);
    for (i = 0, index = 0; i < size; i++) {
        if (carray[i] > 0) {
            rarray[i] /= carray[i];
            garray[i] /= carray[i];
            barray[i] /= carray[i];
            pixcmapAddColor(cmap, rarray[i], garray[i], barray[i]);
            carray[i] = index + 1;  /* to avoid storing 0 */
            index++;
        }
    }

    pixd = pixCreate(w, h, depth);
    pixSetColormap(pixd, cmap);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            pspixel = lines + j;
            extractRGBValues(*pspixel, &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            switch (depth)
            {
            case 2:
                SET_DATA_DIBIT(lined, j, carray[octindex] - 1);
                break;
            case 4:
                SET_DATA_QBIT(lined, j, carray[octindex] - 1);
                break;
            case 8:
                SET_DATA_BYTE(lined, j, carray[octindex] - 1);
                break;
            default:
                L_WARNING("shouldn't get here\n", procName);
            }
        }
    }

array_cleanup:
    LEPT_FREE(carray);
    LEPT_FREE(rarray);
    LEPT_FREE(garray);
    LEPT_FREE(barray);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    return pixd;
}


pixFixedOctcubeQuantGenRGB(PIX     *pixs,
                           l_int32  level)
{
l_int32    w, h, wpls, wpld, i, j;
l_int32    rval, gval, bval;
l_uint32   octindex;
l_uint32  *rtab, *gtab, *btab;
l_uint32  *lines, *lined, *datas, *datad;
PIX       *pixd;

    PROCNAME("pixFixedOctcubeQuantGenRGB");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (level < 1 || level > 6)
        return (PIX *)ERROR_PTR("level not in {1,...6}", procName, NULL);

    if (makeRGBToIndexTables(level, &rtab, &gtab, &btab))
        return (PIX *)ERROR_PTR("tables not made", procName, NULL);

    pixGetDimensions(pixs, &w, &h, NULL);
    pixd = pixCreate(w, h, 32);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            extractRGBValues(lines[j], &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            getRGBFromOctcube(octindex, level, &rval, &gval, &bval);
            composeRGBPixel(rval, gval, bval, lined + j);
        }
    }

    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    return pixd;
}


pixOctcubeQuantFromCmap(PIX      *pixs,
                        PIXCMAP  *cmap,
                        l_int32   mindepth,
                        l_int32   level,
                        l_int32   metric)
{
l_int32   *cmaptab;
l_uint32  *rtab, *gtab, *btab;
PIX       *pixd;

    PROCNAME("pixOctcubeQuantFromCmap");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (!cmap)
        return (PIX *)ERROR_PTR("cmap not defined", procName, NULL);
    if (mindepth != 2 && mindepth != 4 && mindepth != 8)
        return (PIX *)ERROR_PTR("invalid mindepth", procName, NULL);
    if (level < 1 || level > 6)
        return (PIX *)ERROR_PTR("level not in {1...6}", procName, NULL);
    if (metric != L_MANHATTAN_DISTANCE && metric != L_EUCLIDEAN_DISTANCE)
        return (PIX *)ERROR_PTR("invalid metric", procName, NULL);

        /* Set up the tables to map rgb to the nearest colormap index */
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(level, &rtab, &gtab, &btab);
    cmaptab = pixcmapToOctcubeLUT(cmap, level, metric);

    pixd = pixOctcubeQuantFromCmapLUT(pixs, cmap, mindepth,
                                      cmaptab, rtab, gtab, btab);

    LEPT_FREE(cmaptab);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    return pixd;
}


pixRemoveUnusedColors(PIX  *pixs)
{
l_int32     i, j, w, h, d, nc, wpls, val, newval, index, zerofound;
l_int32     rval, gval, bval;
l_uint32   *datas, *lines;
l_int32    *histo, *map1, *map2;
PIXCMAP    *cmap, *cmapd;

    PROCNAME("pixRemoveUnusedColors");

    if (!pixs)
        return ERROR_INT("pixs not defined", procName, 1);
    if ((cmap = pixGetColormap(pixs)) == NULL)
        return 0;

    d = pixGetDepth(pixs);
    if (d != 2 && d != 4 && d != 8)
        return ERROR_INT("d not in {2, 4, 8}", procName, 1);

        /* Find which indices are actually used */
    nc = pixcmapGetCount(cmap);
    if ((histo = (l_int32 *)LEPT_CALLOC(nc, sizeof(l_int32))) == NULL)
        return ERROR_INT("histo not made", procName, 1);
    pixGetDimensions(pixs, &w, &h, NULL);
    wpls = pixGetWpl(pixs);
    datas = pixGetData(pixs);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j++) {
            switch (d)
            {
            case 2:
                val = GET_DATA_DIBIT(lines, j);
                break;
            case 4:
                val = GET_DATA_QBIT(lines, j);
                break;
            case 8:
                val = GET_DATA_BYTE(lines, j);
                break;
            default:
                LEPT_FREE(histo);
                return ERROR_INT("switch ran off end!", procName, 1);
            }
            if (val >= nc) {
                L_WARNING("cmap index out of bounds!\n", procName);
                continue;
            }
            histo[val]++;
        }
    }

        /* Check if there are any zeroes.  If none, quit. */
    zerofound = FALSE;
    for (i = 0; i < nc; i++) {
        if (histo[i] == 0) {
            zerofound = TRUE;
            break;
        }
    }
    if (!zerofound) {
      LEPT_FREE(histo);
      return 0;
    }

        /* Generate mapping tables between indices */
    map1 = (l_int32 *)LEPT_CALLOC(nc, sizeof(l_int32));
    map2 = (l_int32 *)LEPT_CALLOC(nc, sizeof(l_int32));
    index = 0;
    for (i = 0; i < nc; i++) {
        if (histo[i] != 0) {
            map1[index] = i;  /* get old index from new */
            map2[i] = index;  /* get new index from old */
            index++;
        }
    }

        /* Generate new colormap and attach to pixs */
    cmapd = pixcmapCreate(d);
    for (i = 0; i < index; i++) {
        pixcmapGetColor(cmap, map1[i], &rval, &gval, &bval);
        pixcmapAddColor(cmapd, rval, gval, bval);
    }
    pixSetColormap(pixs, cmapd);

        /* Map pixel (index) values to new cmap */
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        for (j = 0; j < w; j++) {
            switch (d)
            {
            case 2:
                val = GET_DATA_DIBIT(lines, j);
                newval = map2[val];
                SET_DATA_DIBIT(lines, j, newval);
                break;
            case 4:
                val = GET_DATA_QBIT(lines, j);
                newval = map2[val];
                SET_DATA_QBIT(lines, j, newval);
                break;
            case 8:
                val = GET_DATA_BYTE(lines, j);
                newval = map2[val];
                SET_DATA_BYTE(lines, j, newval);
                break;
            default:
                LEPT_FREE(histo);
                LEPT_FREE(map1);
                LEPT_FREE(map2);
                return ERROR_INT("switch ran off end!", procName, 1);
            }
        }
    }

    LEPT_FREE(histo);
    LEPT_FREE(map1);
    LEPT_FREE(map2);
    return 0;
}


pixOctcubeQuantFromCmapLUT(PIX       *pixs,
                           PIXCMAP   *cmap,
                           l_int32    mindepth,
                           l_int32   *cmaptab,
                           l_uint32  *rtab,
                           l_uint32  *gtab,
                           l_uint32  *btab)
{
l_int32    i, j, w, h, depth, wpls, wpld;
l_int32    rval, gval, bval, index;
l_uint32   octindex;
l_uint32  *lines, *lined, *datas, *datad;
PIX       *pixd;
PIXCMAP   *cmapc;

    PROCNAME("pixOctcubeQuantFromCmapLUT");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (!cmap)
        return (PIX *)ERROR_PTR("cmap not defined", procName, NULL);
    if (mindepth != 2 && mindepth != 4 && mindepth != 8)
        return (PIX *)ERROR_PTR("invalid mindepth", procName, NULL);
    if (!rtab || !gtab || !btab || !cmaptab)
        return (PIX *)ERROR_PTR("tables not all defined", procName, NULL);

        /* Init dest pix (with minimum bpp depending on cmap) */
    pixcmapGetMinDepth(cmap, &depth);
    depth = L_MAX(depth, mindepth);
    pixGetDimensions(pixs, &w, &h, NULL);
    if ((pixd = pixCreate(w, h, depth)) == NULL)
        return (PIX *)ERROR_PTR("pixd not made", procName, NULL);
    cmapc = pixcmapCopy(cmap);
    pixSetColormap(pixd, cmapc);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);

        /* Insert the colormap index of the color nearest to the input pixel */
    datas = pixGetData(pixs);
    datad = pixGetData(pixd);
    wpls = pixGetWpl(pixs);
    wpld = pixGetWpl(pixd);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            extractRGBValues(lines[j], &rval, &gval, &bval);
                /* Map from rgb to octcube index */
            getOctcubeIndexFromRGB(rval, gval, bval, rtab, gtab, btab,
                                   &octindex);
                /* Map from octcube index to nearest colormap index */
            index = cmaptab[octindex];
            if (depth == 2)
                SET_DATA_DIBIT(lined, j, index);
            else if (depth == 4)
                SET_DATA_QBIT(lined, j, index);
            else  /* depth == 8 */
                SET_DATA_BYTE(lined, j, index);
        }
    }

    return pixd;
}


pixFewColorsOctcubeQuantMixed(PIX       *pixs,
                              l_int32    level,
                              l_int32    darkthresh,
                              l_int32    lightthresh,
                              l_int32    diffthresh,
                              l_float32  minfract,
                              l_int32    maxspan)
{
l_int32    i, j, w, h, wplc, wplm, wpld, ncolors, index;
l_int32    rval, gval, bval, val, minval, maxval;
l_int32   *lut;
l_uint32  *datac, *datam, *datad, *linec, *linem, *lined;
PIX       *pix1, *pixc, *pixm, *pixg, *pixd;
PIXCMAP   *cmap, *cmapd;

    PROCNAME("pixFewColorsOctcubeQuantMixed");

    if (!pixs || pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs undefined or not 32 bpp", procName, NULL);
    if (level <= 0) level = 3;
    if (level > 6)
        return (PIX *)ERROR_PTR("invalid level", procName, NULL);
    if (darkthresh <= 0) darkthresh = 20;
    if (lightthresh <= 0) lightthresh = 244;
    if (diffthresh <= 0) diffthresh = 20;
    if (minfract <= 0.0) minfract = 0.05;
    if (maxspan <= 2) maxspan = 15;

        /* Start with a simple fixed octcube quantizer. */
    if ((pix1 = pixFewColorsOctcubeQuant1(pixs, level)) == NULL)
        return (PIX *)ERROR_PTR("too many colors", procName, NULL);
    pixc = pixConvertTo8(pix1, 1);  /* must be 8 bpp */
    pixDestroy(&pix1);

        /* Identify and save color entries in the colormap.  Set up a LUT
         * that returns -1 for any gray pixel. */
    cmap = pixGetColormap(pixc);
    ncolors = pixcmapGetCount(cmap);
    cmapd = pixcmapCreate(8);
    lut = (l_int32 *)LEPT_CALLOC(256, sizeof(l_int32));
    for (i = 0; i < 256; i++)
        lut[i] = -1;
    for (i = 0, index = 0; i < ncolors; i++) {
        pixcmapGetColor(cmap, i, &rval, &gval, &bval);
        minval = L_MIN(rval, gval);
        minval = L_MIN(minval, bval);
        if (minval > lightthresh)  /* near white */
            continue;
        maxval = L_MAX(rval, gval);
        maxval = L_MAX(maxval, bval);
        if (maxval < darkthresh)  /* near black */
            continue;

            /* Use the max diff between components to test for color */
        if (maxval - minval >= diffthresh) {
            pixcmapAddColor(cmapd, rval, gval, bval);
            lut[i] = index;
            index++;
        }
    }

        /* Generate dest pix with just the color pixels set to their
         * colormap indices.  At the same time, make a 1 bpp mask
         * of the non-color pixels */
    pixGetDimensions(pixs, &w, &h, NULL);
    pixd = pixCreate(w, h, 8);
    pixSetColormap(pixd, cmapd);
    pixm = pixCreate(w, h, 1);
    datac = pixGetData(pixc);
    datam = pixGetData(pixm);
    datad = pixGetData(pixd);
    wplc = pixGetWpl(pixc);
    wplm = pixGetWpl(pixm);
    wpld = pixGetWpl(pixd);
    for (i = 0; i < h; i++) {
        linec = datac + i * wplc;
        linem = datam + i * wplm;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            val = GET_DATA_BYTE(linec, j);
            if (lut[val] == -1)
                SET_DATA_BIT(linem, j);
            else
                SET_DATA_BYTE(lined, j, lut[val]);
        }
    }

        /* Fill in the gray values.  Use a grayscale version of pixs
         * as input, along with the mask over the actual gray pixels. */
    pixg = pixConvertTo8(pixs, 0);
    pixGrayQuantFromHisto(pixd, pixg, pixm, minfract, maxspan);

    LEPT_FREE(lut);
    pixDestroy(&pixc);
    pixDestroy(&pixm);
    pixDestroy(&pixg);
    return pixd;
}


getRGBFromOctcube(l_int32   cubeindex,
                  l_int32   level,
                  l_int32  *prval,
                  l_int32  *pgval,
                  l_int32  *pbval)
{
l_int32  rgbindex;

        /* Bring to format in 21 bits: (r7 g7 b7 r6 g6 b6 ...) */
        /* This is valid for levels from 0 to 6 */
    rgbindex = cubeindex << (3 * (7 - level));  /* upper corner of cube */
    rgbindex |= (0x7 << (3 * (6 - level)));   /* index to center of cube */

        /* Extract separate pieces */
    *prval = ((rgbindex >> 13) & 0x80) |
             ((rgbindex >> 11) & 0x40) |
             ((rgbindex >> 9) & 0x20) |
             ((rgbindex >> 7) & 0x10) |
             ((rgbindex >> 5) & 0x08) |
             ((rgbindex >> 3) & 0x04) |
             ((rgbindex >> 1) & 0x02);
    *pgval = ((rgbindex >> 12) & 0x80) |
             ((rgbindex >> 10) & 0x40) |
             ((rgbindex >> 8) & 0x20) |
             ((rgbindex >> 6) & 0x10) |
             ((rgbindex >> 4) & 0x08) |
             ((rgbindex >> 2) & 0x04) |
             (rgbindex & 0x02);
    *pbval = ((rgbindex >> 11) & 0x80) |
             ((rgbindex >> 9) & 0x40) |
             ((rgbindex >> 7) & 0x20) |
             ((rgbindex >> 5) & 0x10) |
             ((rgbindex >> 3) & 0x08) |
             ((rgbindex >> 1) & 0x04) |
             ((rgbindex << 1) & 0x02);

    return;
}


pixcmapToOctcubeLUT(PIXCMAP  *cmap,
                    l_int32   level,
                    l_int32   metric)
{
l_int32    i, k, size, ncolors, mindist, dist, mincolor, index;
l_int32    rval, gval, bval;  /* color at center of the octcube */
l_int32   *rmap, *gmap, *bmap, *tab;

    PROCNAME("pixcmapToOctcubeLUT");

    if (!cmap)
        return (l_int32 *)ERROR_PTR("cmap not defined", procName, NULL);
    if (level < 1 || level > 6)
        return (l_int32 *)ERROR_PTR("level not in {1...6}", procName, NULL);
    if (metric != L_MANHATTAN_DISTANCE && metric != L_EUCLIDEAN_DISTANCE)
        return (l_int32 *)ERROR_PTR("invalid metric", procName, NULL);

    if (octcubeGetCount(level, &size))  /* array size = 2 ** (3 * level) */
        return (l_int32 *)ERROR_PTR("size not returned", procName, NULL);
    if ((tab = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32))) == NULL)
        return (l_int32 *)ERROR_PTR("tab not allocated", procName, NULL);

    ncolors = pixcmapGetCount(cmap);
    pixcmapToArrays(cmap, &rmap, &gmap, &bmap, NULL);

        /* Assign based on the closest octcube center to the cmap color */
    for (i = 0; i < size; i++) {
        getRGBFromOctcube(i, level, &rval, &gval, &bval);
        mindist = 1000000;
        mincolor = 0;  /* irrelevant init */
        for (k = 0; k < ncolors; k++) {
            if (metric == L_MANHATTAN_DISTANCE) {
                dist = L_ABS(rval - rmap[k]) + L_ABS(gval - gmap[k]) +
                       L_ABS(bval - bmap[k]);
            } else {  /* L_EUCLIDEAN_DISTANCE */
                dist = (rval - rmap[k]) * (rval - rmap[k]) +
                       (gval - gmap[k]) * (gval - gmap[k]) +
                       (bval - bmap[k]) * (bval - bmap[k]);
            }
            if (dist < mindist) {
                mindist = dist;
                mincolor = k;
            }
        }
        tab[i] = mincolor;
    }

        /* Reset black and white if available in the colormap.
         * The darkest octcube is at octindex 0.
         * The lightest octcube is at the max octindex. */
    pixcmapGetNearestIndex(cmap, 0, 0, 0, &index);
    pixcmapGetColor(cmap, index, &rval, &gval, &bval);
    if (rval < 7 && gval < 7 && bval < 7) {
        tab[0] = index;
    }
    pixcmapGetNearestIndex(cmap, 255, 255, 255, &index);
    pixcmapGetColor(cmap, index, &rval, &gval, &bval);
    if (rval > 248 && gval > 248 && bval > 248) {
        tab[(1 << (3 * level)) - 1] = index;
    }

    LEPT_FREE(rmap);
    LEPT_FREE(gmap);
    LEPT_FREE(bmap);
    return tab;
}


pixOctcubeQuantMixedWithGray(PIX     *pixs,
                             l_int32  depth,
                             l_int32  graylevels,
                             l_int32  delta)
{
l_int32    w, h, wpls, wpld, i, j, size, octlevels;
l_int32    rval, gval, bval, del, val, midval;
l_int32   *carray, *rarray, *garray, *barray;
l_int32   *tabval;
l_uint32   octindex;
l_uint32  *rtab, *gtab, *btab;
l_uint32  *lines, *lined, *datas, *datad;
PIX       *pixd;
PIXCMAP   *cmap;

    PROCNAME("pixOctcubeQuantMixedWithGray");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (graylevels < 2)
        return (PIX *)ERROR_PTR("invalid graylevels", procName, NULL);
    if (depth == 4) {
        octlevels = 1;
        size = 8;   /* 2 ** 3 */
        if (graylevels > 8)
            return (PIX *)ERROR_PTR("max 8 gray levels", procName, NULL);
    } else if (depth == 8) {
        octlevels = 2;
        size = 64;   /* 2 ** 6 */
        if (graylevels > 192)
            return (PIX *)ERROR_PTR("max 192 gray levels", procName, NULL);
    } else {
        return (PIX *)ERROR_PTR("output depth not 4 or 8 bpp", procName, NULL);
    }

    pixd = NULL;

        /* Make octcube index tables */
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(octlevels, &rtab, &gtab, &btab);

        /* Make octcube arrays for storing points in each cube */
    carray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    rarray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    garray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));
    barray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32));

        /* Make lookup table, using computed thresholds  */
    tabval = makeGrayQuantIndexTable(graylevels);
    if (!rtab || !gtab || !btab ||
        !carray || !rarray || !garray || !barray || !tabval) {
        L_ERROR("calloc fail for an array\n", procName);
        goto array_cleanup;
    }

        /* Make colormapped output pixd */
    pixGetDimensions(pixs, &w, &h, NULL);
    if ((pixd = pixCreate(w, h, depth)) == NULL) {
        L_ERROR("pixd not made\n", procName);
        goto array_cleanup;
    }
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    cmap = pixcmapCreate(depth);
    for (j = 0; j < size; j++)  /* reserve octcube colors */
        pixcmapAddColor(cmap, 1, 1, 1);  /* a color that won't be used */
    for (j = 0; j < graylevels; j++) {  /* set grayscale colors */
        val = (255 * j) / (graylevels - 1);
        pixcmapAddColor(cmap, val, val, val);
    }
    pixSetColormap(pixd, cmap);
    wpld = pixGetWpl(pixd);
    datad = pixGetData(pixd);

        /* Go through src image: assign dest pixels to colormap values
         * and compute average colors in each occupied octcube */
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            extractRGBValues(lines[j], &rval, &gval, &bval);
            if (rval > gval) {
                if (gval > bval) {   /* r > g > b */
                    del = rval - bval;
                    midval = gval;
                } else if (rval > bval) {  /* r > b > g */
                    del = rval - gval;
                    midval = bval;
                } else {  /* b > r > g */
                    del = bval - gval;
                    midval = rval;
                }
            } else {  /* gval >= rval */
                if (rval > bval) {  /* g > r > b */
                    del = gval - bval;
                    midval = rval;
                } else if (gval > bval) {  /* g > b > r */
                    del = gval - rval;
                    midval = bval;
                } else {  /* b > g > r */
                    del = bval - rval;
                    midval = gval;
                }
            }
            if (del > delta) {  /* assign to color */
                octindex = rtab[rval] | gtab[gval] | btab[bval];
                carray[octindex]++;
                rarray[octindex] += rval;
                garray[octindex] += gval;
                barray[octindex] += bval;
                if (depth == 4)
                    SET_DATA_QBIT(lined, j, octindex);
                else  /* depth == 8 */
                    SET_DATA_BYTE(lined, j, octindex);
            } else {  /* assign to grayscale */
                val = size + tabval[midval];
                if (depth == 4)
                    SET_DATA_QBIT(lined, j, val);
                else  /* depth == 8 */
                    SET_DATA_BYTE(lined, j, val);
            }
        }
    }

        /* Average the colors in each bin and reset the colormap */
    for (i = 0; i < size; i++) {
        if (carray[i] > 0) {
            rarray[i] /= carray[i];
            garray[i] /= carray[i];
            barray[i] /= carray[i];
            pixcmapResetColor(cmap, i, rarray[i], garray[i], barray[i]);
        }
    }

array_cleanup:
    LEPT_FREE(carray);
    LEPT_FREE(rarray);
    LEPT_FREE(garray);
    LEPT_FREE(barray);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    LEPT_FREE(tabval);

    return pixd;
}


octreeFindColorCell(l_int32    octindex,
                    CQCELL  ***cqcaa,
                    l_int32   *pindex,
                    l_int32   *prval,
                    l_int32   *pgval,
                    l_int32   *pbval)
{
l_int32  level;
l_int32  baseindex, subindex;
CQCELL  *cqc, *cqcsub;

        /* Use rgb values stored in the cubes; a little faster */
    for (level = 2; level < CqNLevels; level++) {
        getOctcubeIndices(octindex, level, &baseindex, &subindex);
        cqc = cqcaa[level][baseindex];
        cqcsub = cqcaa[level + 1][subindex];
        if (cqcsub->bleaf == 0) {  /* use cell at level above */
            *pindex = cqc->index;
            *prval = cqc->rc;
            *pgval = cqc->gc;
            *pbval = cqc->bc;
            break;
        } else if (level == CqNLevels - 1) {  /* reached the bottom */
            *pindex = cqcsub->index;
            *prval = cqcsub->rc;
            *pgval = cqcsub->gc;
            *pbval = cqcsub->bc;
             break;
        }
    }

#if 0
        /* Generate rgb values for each cube on the fly; slower */
    for (level = 2; level < CqNLevels; level++) {
        l_int32  rv, gv, bv;
        getOctcubeIndices(octindex, level, &baseindex, &subindex);
        cqc = cqcaa[level][baseindex];
        cqcsub = cqcaa[level + 1][subindex];
        if (cqcsub->bleaf == 0) {  /* use cell at level above */
            getRGBFromOctcube(baseindex, level, &rv, &gv, &bv);
            *pindex = cqc->index;
            *prval = rv;
            *pgval = gv;
            *pbval = bv;
            break;
        } else if (level == CqNLevels - 1) {  /* reached the bottom */
            getRGBFromOctcube(subindex, level + 1, &rv, &gv, &bv);
           *pindex = cqcsub->index;
            *prval = rv;
            *pgval = gv;
            *pbval = bv;
            break;
        }
    }
#endif

    return 0;
}


cqcellTreeDestroy(CQCELL  ****pcqcaa)
{
l_int32    level, ncells, i;
CQCELL  ***cqcaa;
CQCELL   **cqca;

    PROCNAME("cqcellTreeDestroy");

    if (pcqcaa == NULL) {
        L_WARNING("ptr address is NULL\n", procName);
        return;
    }

    if ((cqcaa = *pcqcaa) == NULL)
        return;

    for (level = 0; level <= CqNLevels; level++) {
        cqca = cqcaa[level];
        ncells = 1 << (3 * level);
        for (i = 0; i < ncells; i++)
            LEPT_FREE(cqca[i]);
        LEPT_FREE(cqca);
    }
    LEPT_FREE(cqcaa);
    *pcqcaa = NULL;

    return;
}


pixOctreeColorQuant(PIX     *pixs,
                    l_int32  colors,
                    l_int32  ditherflag)
{
    PROCNAME("pixOctreeColorQuant");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (colors < 128 || colors > 240)  /* further restricted */
        return (PIX *)ERROR_PTR("colors must be in [128, 240]", procName, NULL);

    return pixOctreeColorQuantGeneral(pixs, colors, ditherflag, 0.01, 0.01);
}


pixOctreeColorQuantGeneral(PIX       *pixs,
                           l_int32    colors,
                           l_int32    ditherflag,
                           l_float32  validthresh,
                           l_float32  colorthresh)
{
l_int32    w, h, minside, factor, index, rval, gval, bval;
l_float32  scalefactor;
l_float32  pixfract;  /* fraction neither near white nor black */
l_float32  colorfract;  /* fraction with color of the pixfract population */
CQCELL  ***cqcaa;
PIX       *pixd, *pixsub;
PIXCMAP   *cmap;

    PROCNAME("pixOctreeColorQuantGeneral");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (colors < 128 || colors > 240)
        return (PIX *)ERROR_PTR("colors must be in [128, 240]", procName, NULL);

        /* Determine if the image has sufficient color content for
         *   octree quantization, based on the input thresholds.
         * If pixfract << 1, most pixels are close to black or white.
         * If colorfract << 1, the pixels that are not near
         *   black or white have very little color.
         * If with insufficient color, quantize with a grayscale colormap. */
    pixGetDimensions(pixs, &w, &h, NULL);
    if (validthresh > 0.0 && colorthresh > 0.0) {
        minside = L_MIN(w, h);
        factor = L_MAX(1, minside / 400);
        pixColorFraction(pixs, 20, 244, 20, factor, &pixfract, &colorfract);
        if (pixfract * colorfract < validthresh * colorthresh) {
            L_INFO("\n  Pixel fraction neither white nor black = %6.3f"
                   "\n  Color fraction of those pixels = %6.3f"
                   "\n  Quantizing to 8 bpp gray\n",
                   procName, pixfract, colorfract);
            return pixConvertTo8(pixs, 1);
        }
    } else {
        L_INFO("\n  Process in color by default\n", procName);
    }

        /* Conditionally subsample to speed up the first pass */
    if (w > TreeGenWidth) {
        scalefactor = (l_float32)TreeGenWidth / (l_float32)w;
        pixsub = pixScaleBySampling(pixs, scalefactor, scalefactor);
    } else {
        pixsub = pixClone(pixs);
    }

        /* Drop the number of requested colors if image is very small */
    if (w < MinDitherSize && h < MinDitherSize)
        colors = L_MIN(colors, 220);

        /* Make the pruned octree */
    cqcaa = octreeGenerateAndPrune(pixsub, colors, CqReservedColors, &cmap);
    if (!cqcaa) {
        pixDestroy(&pixsub);
        return (PIX *)ERROR_PTR("tree not made", procName, NULL);
    }
#if DEBUG_COLORQUANT
    L_INFO(" Colors requested = %d\n", procName, colors);
    L_INFO(" Actual colors = %d\n", procName, cmap->n);
#endif  /* DEBUG_COLORQUANT */

        /* Do not dither if image is very small */
    if (w < MinDitherSize && h < MinDitherSize && ditherflag == 1) {
        L_INFO("Small image: dithering turned off\n", procName);
        ditherflag = 0;
    }

        /* Traverse tree from root, looking for lowest cube
         * that is a leaf, and set dest pix value to its
         * colortable index */
    if ((pixd = pixOctreeQuantizePixels(pixs, cqcaa, ditherflag)) == NULL) {
        pixDestroy(&pixsub);
        cqcellTreeDestroy(&cqcaa);
        return (PIX *)ERROR_PTR("pixd not made", procName, NULL);
    }

        /* Attach colormap and copy res */
    pixSetColormap(pixd, cmap);
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);

        /* Force darkest color to black if each component <= 4 */
    pixcmapGetRankIntensity(cmap, 0.0, &index);
    pixcmapGetColor(cmap, index, &rval, &gval, &bval);
    if (rval < 5 && gval < 5 && bval < 5)
        pixcmapResetColor(cmap, index, 0, 0, 0);

        /* Force lightest color to white if each component >= 252 */
    pixcmapGetRankIntensity(cmap, 1.0, &index);
    pixcmapGetColor(cmap, index, &rval, &gval, &bval);
    if (rval > 251 && gval > 251 && bval > 251)
        pixcmapResetColor(cmap, index, 255, 255, 255);

    cqcellTreeDestroy(&cqcaa);
    pixDestroy(&pixsub);
    return pixd;
}


getOctcubeIndices(l_int32   rgbindex,
                  l_int32   level,
                  l_int32  *pbindex,
                  l_int32  *psindex)
{
    PROCNAME("getOctcubeIndex");

    if (level < 0 || level > CqNLevels - 1)
        return ERROR_INT("level must be in e.g., [0 ... 5]", procName, 1);
    if (!pbindex)
        return ERROR_INT("&bindex not defined", procName, 1);
    if (!psindex)
        return ERROR_INT("&sindex not defined", procName, 1);

    *pbindex = rgbindex >> (3 * (CqNLevels - level));
    *psindex = rgbindex >> (3 * (CqNLevels - 1 - level));
    return 0;
}


pixNumberOccupiedOctcubes(PIX       *pix,
                          l_int32    level,
                          l_int32    mincount,
                          l_float32  minfract,
                          l_int32   *pncolors)
{
l_int32    i, j, w, h, d, wpl, ncolors, size, octindex;
l_int32    rval, gval, bval;
l_int32   *carray;
l_uint32  *data, *line, *rtab, *gtab, *btab;

    PROCNAME("pixNumberOccupiedOctcubes");

    if (!pncolors)
        return ERROR_INT("&ncolors not defined", procName, 1);
    *pncolors = 0;
    if (!pix)
        return ERROR_INT("pix not defined", procName, 1);
    pixGetDimensions(pix, &w, &h, &d);
    if (d != 32)
        return ERROR_INT("pix not 32 bpp", procName, 1);
    if (level < 1 || level > 6)
        return ERROR_INT("invalid level", procName, 1);
    if ((mincount < 0 && minfract < 0) || (mincount >= 0.0 && minfract >= 0.0))
        return ERROR_INT("invalid mincount/minfract", procName, 1);
    if (mincount == 0 || minfract == 0.0)
        mincount = 1;
    else if (minfract > 0.0)
        mincount = L_MIN(1, (l_int32)(minfract * w * h));

    if (octcubeGetCount(level, &size))  /* array size = 2 ** (3 * level) */
        return ERROR_INT("size not returned", procName, 1);
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(level, &rtab, &gtab, &btab);
    if ((carray = (l_int32 *)LEPT_CALLOC(size, sizeof(l_int32))) == NULL) {
        L_ERROR("carray not made\n", procName);
        goto cleanup_arrays;
    }

        /* Mark the occupied octcube leaves */
    data = pixGetData(pix);
    wpl = pixGetWpl(pix);
    for (i = 0; i < h; i++) {
        line = data + i * wpl;
        for (j = 0; j < w; j++) {
            extractRGBValues(line[j], &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            carray[octindex]++;
        }
    }

        /* Count them */
    for (i = 0, ncolors = 0; i < size; i++) {
        if (carray[i] >= mincount)
            ncolors++;
    }
    *pncolors = ncolors;

cleanup_arrays:
    LEPT_FREE(carray);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);
    return 0;
}


pixFewColorsOctcubeQuant2(PIX      *pixs,
                          l_int32   level,
                          NUMA     *na,
                          l_int32   ncolors,
                          l_int32  *pnerrors)
{
l_int32    w, h, wpls, wpld, i, j, nerrors;
l_int32    ncubes, depth, cindex, oval;
l_int32    rval, gval, bval;
l_int32   *octarray;
l_uint32   octindex;
l_uint32  *rtab, *gtab, *btab;
l_uint32  *lines, *lined, *datas, *datad, *ppixel;
l_uint32  *colorarray;
PIX       *pixd;
PIXCMAP   *cmap;

    PROCNAME("pixFewColorsOctcubeQuant2");

    if (!pixs)
        return (PIX *)ERROR_PTR("pixs not defined", procName, NULL);
    if (pixGetDepth(pixs) != 32)
        return (PIX *)ERROR_PTR("pixs not 32 bpp", procName, NULL);
    if (level < 3 || level > 6)
        return (PIX *)ERROR_PTR("level not in {4, 5, 6}", procName, NULL);
    if (ncolors > 256)
        return (PIX *)ERROR_PTR("ncolors > 256", procName, NULL);
    if (pnerrors)
        *pnerrors = UNDEF;

    pixd = NULL;

        /* Represent the image with a set of leaf octcubes
         * at 'level', one for each color. */
    rtab = gtab = btab = NULL;
    makeRGBToIndexTables(level, &rtab, &gtab, &btab);

        /* The octarray will give a ptr from the octcube to the colorarray */
    ncubes = numaGetCount(na);
    octarray = (l_int32 *)LEPT_CALLOC(ncubes, sizeof(l_int32));

        /* The colorarray will hold the colors of the first pixel
         * that lands in the leaf octcube.  After filling, it is
         * used to generate the colormap.  */
    colorarray = (l_uint32 *)LEPT_CALLOC(ncolors + 1, sizeof(l_uint32));
    if (!octarray || !colorarray) {
        L_ERROR("octarray or colorarray not made\n", procName);
        goto cleanup_arrays;
    }

        /* Determine the output depth from the number of colors */
    pixGetDimensions(pixs, &w, &h, NULL);
    datas = pixGetData(pixs);
    wpls = pixGetWpl(pixs);
    if (ncolors <= 4)
        depth = 2;
    else if (ncolors <= 16)
        depth = 4;
    else  /* ncolors <= 256 */
        depth = 8;

    if ((pixd = pixCreate(w, h, depth)) == NULL) {
        L_ERROR("pixd not made\n", procName);
        goto cleanup_arrays;
    }
    pixCopyResolution(pixd, pixs);
    pixCopyInputFormat(pixd, pixs);
    datad = pixGetData(pixd);
    wpld = pixGetWpl(pixd);

        /* For each pixel, get the octree index for its leaf octcube.
         * Check if a pixel has already been found in this octcube.
         *   ~ If not yet found, save that color in the colorarray
         *     and save the cindex in the octarray.
         *   ~ If already found, compare the pixel color with the
         *     color in the colorarray, and note if it differs.
         * Then set the dest pixel value to the cindex - 1, which
         * will be the cmap index for this color.  */
    cindex = 1;  /* start with 1 */
    nerrors = 0;
    for (i = 0; i < h; i++) {
        lines = datas + i * wpls;
        lined = datad + i * wpld;
        for (j = 0; j < w; j++) {
            ppixel = lines + j;
            extractRGBValues(*ppixel, &rval, &gval, &bval);
            octindex = rtab[rval] | gtab[gval] | btab[bval];
            oval = octarray[octindex];
            if (oval == 0) {
                octarray[octindex] = cindex;
                colorarray[cindex] = *ppixel;
                setPixelLow(lined, j, depth, cindex - 1);
                cindex++;
            } else {  /* already have seen this color; is it unique? */
                setPixelLow(lined, j, depth, oval - 1);
                if (colorarray[oval] != *ppixel)
                    nerrors++;
            }
        }
    }
    if (pnerrors)
        *pnerrors = nerrors;

#if  DEBUG_FEW_COLORS
    lept_stderr("ncubes = %d, ncolors = %d\n", ncubes, ncolors);
    for (i = 0; i < ncolors; i++)
        lept_stderr("color[%d] = %x\n", i, colorarray[i + 1]);
#endif  /* DEBUG_FEW_COLORS */

        /* Make the colormap. */
    cmap = pixcmapCreate(depth);
    for (i = 0; i < ncolors; i++) {
        ppixel = colorarray + i + 1;
        extractRGBValues(*ppixel, &rval, &gval, &bval);
        pixcmapAddColor(cmap, rval, gval, bval);
    }
    pixSetColormap(pixd, cmap);

cleanup_arrays:
    LEPT_FREE(octarray);
    LEPT_FREE(colorarray);
    LEPT_FREE(rtab);
    LEPT_FREE(gtab);
    LEPT_FREE(btab);

    return pixd;
}