feat: support image background editor [wip]

src/actions/actionFinalize.tsx

@@ -14,10 +14,60 @@ import {
   bindOrUnbindLinearElement,
 } from "../element/binding";
 import { isBindingElement } from "../element/typeChecks";
+import { ExcalidrawImageElement } from "../element/types";
+import { imageFromImageData } from "../element/image";
 
 export const actionFinalize = register({
   name: "finalize",
-  perform: (elements, appState, _, { canvas, focusContainer }) => {
+  perform: (
+    elements,
+    appState,
+    _,
+    { canvas, focusContainer, imageCache, addFiles },
+  ) => {
+    if (appState.editingImageElement) {
+      const { elementId, imageData } = appState.editingImageElement;
+      const editingImageElement = elements.find((el) => el.id === elementId) as
+        | ExcalidrawImageElement
+        | undefined;
+      if (editingImageElement?.fileId) {
+        const cachedImageData = imageCache.get(editingImageElement.fileId);
+        if (cachedImageData) {
+          const { image, dataURL } = imageFromImageData(imageData);
+
+          imageCache.set(editingImageElement.fileId, {
+            ...cachedImageData,
+            image,
+          });
+
+          addFiles([
+            {
+              id: editingImageElement.fileId,
+              dataURL,
+              mimeType: cachedImageData.mimeType,
+              created: Date.now(),
+            },
+          ]);
+
+          return {
+            appState: {
+              ...appState,
+              editingImageElement: null,
+            },
+            commitToHistory: false,
+          };
+        }
+      }
+
+      return {
+        appState: {
+          ...appState,
+          editingImageElement: null,
+        },
+        commitToHistory: false,
+      };
+    }
+
     if (appState.editingLinearElement) {
       const { elementId, startBindingElement, endBindingElement } =
         appState.editingLinearElement;
@@ -162,6 +212,7 @@ export const actionFinalize = register({
   keyTest: (event, appState) =>
     (event.key === KEYS.ESCAPE &&
       (appState.editingLinearElement !== null ||
+        appState.editingImageElement !== null ||
         (!appState.draggingElement && appState.multiElement === null))) ||
     ((event.key === KEYS.ESCAPE || event.key === KEYS.ENTER) &&
       appState.multiElement !== null),

src/actions/actionImageEditing.tsx

@@ -0,0 +1,75 @@
+import { getSelectedElements, isSomeElementSelected } from "../scene";
+import { ToolButton } from "../components/ToolButton";
+import { backgroundIcon } from "../components/icons";
+import { register } from "./register";
+import { getNonDeletedElements } from "../element";
+import { isInitializedImageElement } from "../element/typeChecks";
+import Scene from "../scene/Scene";
+
+export const actionEditImageAlpha = register({
+  name: "editImageAlpha",
+  perform: async (elements, appState, _, app) => {
+    if (appState.editingImageElement) {
+      return {
+        appState: {
+          ...appState,
+          editingImageElement: null,
+        },
+        commitToHistory: false,
+      };
+    }
+
+    const selectedElements = getSelectedElements(elements, appState);
+    const selectedElement = selectedElements[0];
+    if (
+      selectedElements.length === 1 &&
+      isInitializedImageElement(selectedElement)
+    ) {
+      const imgData = app.imageCache.get(selectedElement.fileId);
+      if (!imgData) {
+        return false;
+      }
+
+      const image = await imgData.image;
+      const { width, height } = image;
+
+      const canvas = document.createElement("canvas");
+      canvas.height = height;
+      canvas.width = width;
+      const context = canvas.getContext("2d")!;
+
+      context.drawImage(image, 0, 0, width, height);
+
+      const imageData = context.getImageData(0, 0, width, height);
+
+      Scene.mapElementToScene(selectedElement.id, app.scene);
+
+      return {
+        appState: {
+          ...appState,
+          editingImageElement: {
+            editorType: "alpha",
+            elementId: selectedElement.id,
+            origImageData: imageData,
+            imageData,
+            pointerDownState: { screenX: 0, screenY: 0, sampledPixel: null },
+          },
+        },
+        commitToHistory: false,
+      };
+    }
+    return false;
+  },
+  PanelComponent: ({ elements, appState, updateData }) => (
+    <ToolButton
+      type="button"
+      icon={backgroundIcon}
+      label="Edit Image Alpha"
+      className={appState.editingImageElement ? "active" : ""}
+      title={"Edit image alpha"}
+      aria-label={"Edit image alpha"}
+      onClick={() => updateData(null)}
+      visible={isSomeElementSelected(getNonDeletedElements(elements), appState)}
+    />
+  ),
+});

src/actions/index.ts

@@ -80,3 +80,4 @@ export { actionToggleGridMode } from "./actionToggleGridMode";
 export { actionToggleZenMode } from "./actionToggleZenMode";
 
 export { actionToggleStats } from "./actionToggleStats";
+export { actionEditImageAlpha } from "./actionImageEditing";

src/actions/types.ts

@@ -101,7 +101,8 @@ export type ActionName =
   | "flipVertical"
   | "viewMode"
   | "exportWithDarkMode"
-  | "toggleTheme";
+  | "toggleTheme"
+  | "editImageAlpha";
 
 export type PanelComponentProps = {
   elements: readonly ExcalidrawElement[];

src/appState.ts

@@ -41,6 +41,7 @@ export const getDefaultAppState = (): Omit<
     editingElement: null,
     editingGroupId: null,
     editingLinearElement: null,
+    editingImageElement: null,
     elementLocked: false,
     elementType: "selection",
     errorMessage: null,
@@ -125,6 +126,7 @@ const APP_STATE_STORAGE_CONF = (<
   editingElement: { browser: false, export: false, server: false },
   editingGroupId: { browser: true, export: false, server: false },
   editingLinearElement: { browser: false, export: false, server: false },
+  editingImageElement: { browser: false, export: false, server: false },
   elementLocked: { browser: true, export: false, server: false },
   elementType: { browser: true, export: false, server: false },
   errorMessage: { browser: false, export: false, server: false },

src/components/Actions.tsx

@@ -19,6 +19,7 @@ import { capitalizeString, isTransparent, setCursorForShape } from "../utils";
 import Stack from "./Stack";
 import { ToolButton } from "./ToolButton";
 import { hasStrokeColor } from "../scene/comparisons";
+import { isImageElement } from "../element/typeChecks";
 
 export const SelectedShapeActions = ({
   appState,
@@ -105,6 +106,13 @@ export const SelectedShapeActions = ({
         <>{renderAction("changeArrowhead")}</>
       )}
 
+      <fieldset>
+        <div className="buttonList">
+          {targetElements.some((element) => isImageElement(element)) &&
+            renderAction("editImageAlpha")}
+        </div>
+      </fieldset>
+
       {renderAction("changeOpacity")}
 
       <fieldset>

src/components/App.tsx

@@ -237,6 +237,7 @@ import {
   getBoundTextElementId,
 } from "../element/textElement";
 import { isHittingElementNotConsideringBoundingBox } from "../element/collision";
+import { ImageEditor } from "../element/imageEditor";
 
 const IsMobileContext = React.createContext(false);
 export const useIsMobile = () => useContext(IsMobileContext);
@@ -281,7 +282,7 @@ class App extends React.Component<AppProps, AppState> {
     UIOptions: DEFAULT_UI_OPTIONS,
   };
 
-  private scene: Scene;
+  public scene: Scene;
   private resizeObserver: ResizeObserver | undefined;
   private nearestScrollableContainer: HTMLElement | Document | undefined;
   public library: AppClassProperties["library"];
@@ -1031,8 +1032,14 @@ class App extends React.Component<AppProps, AppState> {
     );
 
     if (
-      this.state.editingLinearElement &&
+      (this.state.editingLinearElement &&
-      !this.state.selectedElementIds[this.state.editingLinearElement.elementId]
+        !this.state.selectedElementIds[
+          this.state.editingLinearElement.elementId
+        ]) ||
+      (this.state.editingImageElement &&
+        !this.state.selectedElementIds[
+          this.state.editingImageElement.elementId
+        ])
     ) {
       // defer so that the commitToHistory flag isn't reset via current update
       setTimeout(() => {
@@ -1135,6 +1142,7 @@ class App extends React.Component<AppProps, AppState> {
         imageCache: this.imageCache,
         isExporting: false,
         renderScrollbars: !this.isMobile,
+        editingImageElement: this.state.editingImageElement,
       },
     );
     if (scrollBars) {
@@ -2330,6 +2338,10 @@ class App extends React.Component<AppProps, AppState> {
     const scenePointer = viewportCoordsToSceneCoords(event, this.state);
     const { x: scenePointerX, y: scenePointerY } = scenePointer;
 
+    if (this.state.editingImageElement) {
+      return;
+    }
+
     if (
       this.state.editingLinearElement &&
       !this.state.editingLinearElement.isDragging
@@ -2920,6 +2932,14 @@ class App extends React.Component<AppProps, AppState> {
     pointerDownState: PointerDownState,
   ): boolean => {
     if (this.state.elementType === "selection") {
+      if (this.state.editingImageElement) {
+        ImageEditor.handlePointerDown(
+          this.state.editingImageElement,
+          pointerDownState.origin,
+        );
+        return false;
+      }
+
       const elements = this.scene.getElements();
       const selectedElements = getSelectedElements(elements, this.state);
       if (selectedElements.length === 1 && !this.state.editingLinearElement) {
@@ -3480,6 +3500,22 @@ class App extends React.Component<AppProps, AppState> {
         }
       }
 
+      if (this.state.editingImageElement) {
+        const newImageData = ImageEditor.handlePointerMove(
+          this.state.editingImageElement,
+          pointerCoords,
+        );
+        if (newImageData) {
+          this.setState({
+            editingImageElement: {
+              ...this.state.editingImageElement,
+              imageData: newImageData,
+            },
+          });
+        }
+        return;
+      }
+
       if (this.state.editingLinearElement) {
         const didDrag = LinearElementEditor.handlePointDragging(
           this.state,
@@ -3802,6 +3838,10 @@ class App extends React.Component<AppProps, AppState> {
 
       this.savePointer(childEvent.clientX, childEvent.clientY, "up");
 
+      if (this.state.editingImageElement) {
+        ImageEditor.handlePointerUp(this.state.editingImageElement);
+      }
+
       // Handle end of dragging a point of a linear element, might close a loop
       // and sets binding element
       if (this.state.editingLinearElement) {

src/components/icons.tsx

@@ -89,6 +89,14 @@ export const trash = createIcon(
 
   { width: 448, height: 512 },
 );
+export const backgroundIcon = createIcon(
+  <path
+    fill="currentColor"
+    d="M512 320s-64 92.65-64 128c0 35.35 28.66 64 64 64s64-28.65 64-64-64-128-64-128zm-9.37-102.94L294.94 9.37C288.69 3.12 280.5 0 272.31 0s-16.38 3.12-22.62 9.37l-81.58 81.58L81.93 4.76c-6.25-6.25-16.38-6.25-22.62 0L36.69 27.38c-6.24 6.25-6.24 16.38 0 22.62l86.19 86.18-94.76 94.76c-37.49 37.48-37.49 98.26 0 135.75l117.19 117.19c18.74 18.74 43.31 28.12 67.87 28.12 24.57 0 49.13-9.37 67.87-28.12l221.57-221.57c12.5-12.5 12.5-32.75.01-45.25zm-116.22 70.97H65.93c1.36-3.84 3.57-7.98 7.43-11.83l13.15-13.15 81.61-81.61 58.6 58.6c12.49 12.49 32.75 12.49 45.24 0s12.49-32.75 0-45.24l-58.6-58.6 58.95-58.95 162.44 162.44-48.34 48.34z"
+  ></path>,
+
+  { width: 576, height: 512 },
+);
 
 export const palette = createIcon(
   "M204.3 5C104.9 24.4 24.8 104.3 5.2 203.4c-37 187 131.7 326.4 258.8 306.7 41.2-6.4 61.4-54.6 42.5-91.7-23.1-45.4 9.9-98.4 60.9-98.4h79.7c35.8 0 64.8-29.6 64.9-65.3C511.5 97.1 368.1-26.9 204.3 5zM96 320c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm32-128c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128-64c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32zm128 64c-17.7 0-32-14.3-32-32s14.3-32 32-32 32 14.3 32 32-14.3 32-32 32z",

src/element/image.ts

@@ -109,3 +109,16 @@ export const normalizeSVG = async (SVGString: string) => {
     return svg.outerHTML;
   }
 };
+
+export const imageFromImageData = (imagedata: ImageData) => {
+  const canvas = document.createElement("canvas");
+  const ctx = canvas.getContext("2d")!;
+  canvas.width = imagedata.width;
+  canvas.height = imagedata.height;
+  ctx.putImageData(imagedata, 0, 0);
+
+  const image = new Image();
+  const dataURL = canvas.toDataURL() as DataURL;
+  image.src = dataURL;
+  return { image, dataURL };
+};

src/element/imageEditor.ts

@@ -0,0 +1,112 @@
+import { distance2d } from "../math";
+import Scene from "../scene/Scene";
+import {
+  ExcalidrawImageElement,
+  InitializedExcalidrawImageElement,
+} from "./types";
+
+export type EditingImageElement = {
+  editorType: "alpha";
+  elementId: ExcalidrawImageElement["id"];
+  origImageData: Readonly<ImageData>;
+  imageData: ImageData;
+  pointerDownState: {
+    screenX: number;
+    screenY: number;
+    sampledPixel: readonly [number, number, number, number] | null;
+  };
+};
+
+const getElement = (id: EditingImageElement["elementId"]) => {
+  const element = Scene.getScene(id)?.getNonDeletedElement(id);
+  if (element) {
+    return element as InitializedExcalidrawImageElement;
+  }
+  return null;
+};
+
+export class ImageEditor {
+  static handlePointerDown(
+    editingElement: EditingImageElement,
+    scenePointer: { x: number; y: number },
+  ) {
+    const imageElement = getElement(editingElement.elementId);
+
+    if (imageElement) {
+      if (
+        scenePointer.x >= imageElement.x &&
+        scenePointer.x <= imageElement.x + imageElement.width &&
+        scenePointer.y >= imageElement.y &&
+        scenePointer.y <= imageElement.y + imageElement.height
+      ) {
+        editingElement.pointerDownState.screenX = scenePointer.x;
+        editingElement.pointerDownState.screenY = scenePointer.y;
+
+        const { width, height, data } = editingElement.origImageData;
+
+        const imageOffsetX = Math.round(
+          (scenePointer.x - imageElement.x) * (width / imageElement.width),
+        );
+        const imageOffsetY = Math.round(
+          (scenePointer.y - imageElement.y) * (height / imageElement.height),
+        );
+
+        const sampledPixel = [
+          data[(imageOffsetY * width + imageOffsetX) * 4 + 0],
+          data[(imageOffsetY * width + imageOffsetX) * 4 + 1],
+          data[(imageOffsetY * width + imageOffsetX) * 4 + 2],
+          data[(imageOffsetY * width + imageOffsetX) * 4 + 3],
+        ] as const;
+
+        editingElement.pointerDownState.sampledPixel = sampledPixel;
+      }
+    }
+  }
+
+  static handlePointerMove(
+    editingElement: EditingImageElement,
+    scenePointer: { x: number; y: number },
+  ) {
+    const { sampledPixel } = editingElement.pointerDownState;
+    if (sampledPixel) {
+      const { screenX, screenY } = editingElement.pointerDownState;
+      const distance = distance2d(
+        scenePointer.x,
+        scenePointer.y,
+        screenX,
+        screenY,
+      );
+
+      const { width, height, data } = editingElement.origImageData;
+      const newImageData = new ImageData(width, height);
+
+      for (let x = 0; x < width; ++x) {
+        for (let y = 0; y < height; ++y) {
+          if (
+            Math.abs(sampledPixel[0] - data[(y * width + x) * 4 + 0]) +
+              Math.abs(sampledPixel[1] - data[(y * width + x) * 4 + 1]) +
+              Math.abs(sampledPixel[2] - data[(y * width + x) * 4 + 2]) <
+            distance
+          ) {
+            newImageData.data[(y * width + x) * 4 + 0] = 0;
+            newImageData.data[(y * width + x) * 4 + 1] = 255;
+            newImageData.data[(y * width + x) * 4 + 2] = 0;
+            newImageData.data[(y * width + x) * 4 + 3] = 0;
+          } else {
+            for (let p = 0; p < 4; ++p) {
+              newImageData.data[(y * width + x) * 4 + p] =
+                data[(y * width + x) * 4 + p];
+            }
+          }
+        }
+      }
+
+      return newImageData;
+    }
+  }
+
+  static handlePointerUp(editingElement: EditingImageElement) {
+    editingElement.pointerDownState.sampledPixel = null;
+    editingElement.origImageData = editingElement.imageData;
+  }
+}

src/renderer/renderElement.ts

@@ -12,6 +12,7 @@ import {
   isLinearElement,
   isFreeDrawElement,
   isInitializedImageElement,
+  isImageElement,
 } from "../element/typeChecks";
 import {
   getDiamondPoints,
@@ -221,19 +222,31 @@ const drawElementOnCanvas = (
       break;
     }
     case "image": {
-      const img = isInitializedImageElement(element)
+      if (renderConfig.editingImageElement) {
-        ? renderConfig.imageCache.get(element.fileId)?.image
+        const { imageData } = renderConfig.editingImageElement;
-        : undefined;
+
-      if (img != null && !(img instanceof Promise)) {
+        const imgCanvas = document.createElement("canvas");
-        context.drawImage(
+        imgCanvas.width = imageData.width;
-          img,
+        imgCanvas.height = imageData.height;
-          0 /* hardcoded for the selection box*/,
+        const imgContext = imgCanvas.getContext("2d")!;
-          0,
+        imgContext.putImageData(imageData, 0, 0);
-          element.width,
+
-          element.height,
+        context.drawImage(imgCanvas, 0, 0, element.width, element.height);
-        );
       } else {
-        drawImagePlaceholder(element, context, renderConfig.zoom.value);
+        const img = isInitializedImageElement(element)
+          ? renderConfig.imageCache.get(element.fileId)?.image
+          : undefined;
+        if (img != null && !(img instanceof Promise)) {
+          context.drawImage(
+            img,
+            0 /* hardcoded for the selection box*/,
+            0,
+            element.width,
+            element.height,
+          );
+        } else {
+          drawImagePlaceholder(element, context, renderConfig.zoom.value);
+        }
       }
       break;
     }
@@ -410,23 +423,23 @@ const generateElementShape = (
               topY + (rightY - topY) * 0.25
             } L ${rightX - (rightX - topX) * 0.25} ${
               rightY - (rightY - topY) * 0.25
-            } 
+            }
             C ${rightX} ${rightY}, ${rightX} ${rightY}, ${
               rightX - (rightX - bottomX) * 0.25
-            } ${rightY + (bottomY - rightY) * 0.25} 
+            } ${rightY + (bottomY - rightY) * 0.25}
             L ${bottomX + (rightX - bottomX) * 0.25} ${
               bottomY - (bottomY - rightY) * 0.25
-            }  
+            }
             C ${bottomX} ${bottomY}, ${bottomX} ${bottomY}, ${
               bottomX - (bottomX - leftX) * 0.25
-            } ${bottomY - (bottomY - leftY) * 0.25} 
+            } ${bottomY - (bottomY - leftY) * 0.25}
             L ${leftX + (bottomX - leftX) * 0.25} ${
               leftY + (bottomY - leftY) * 0.25
-            } 
+            }
             C ${leftX} ${leftY}, ${leftX} ${leftY}, ${
               leftX + (topX - leftX) * 0.25
-            } ${leftY - (leftY - topY) * 0.25} 
+            } ${leftY - (leftY - topY) * 0.25}
-            L ${topX - (topX - leftX) * 0.25} ${topY + (leftY - topY) * 0.25} 
+            L ${topX - (topX - leftX) * 0.25} ${topY + (leftY - topY) * 0.25}
             C ${topX} ${topY}, ${topX} ${topY}, ${
               topX + (rightX - topX) * 0.25
             } ${topY + (rightY - topY) * 0.25}`,
@@ -608,7 +621,10 @@ const generateElementWithCanvas = (
   if (
     !prevElementWithCanvas ||
     shouldRegenerateBecauseZoom ||
-    prevElementWithCanvas.theme !== renderConfig.theme
+    prevElementWithCanvas.theme !== renderConfig.theme ||
+    (renderConfig.editingImageElement &&
+      isImageElement(element) &&
+      element.id === renderConfig.editingImageElement.elementId)
   ) {
     const elementWithCanvas = generateElementCanvas(
       element,

src/scene/Scene.ts

@@ -4,15 +4,13 @@ import {
   NonDeleted,
 } from "../element/types";
 import { getNonDeletedElements, isNonDeletedElement } from "../element";
-import { LinearElementEditor } from "../element/linearElementEditor";
 
-type ElementIdKey = InstanceType<typeof LinearElementEditor>["elementId"];
+type ElementKey = ExcalidrawElement | ExcalidrawElement["id"];
-type ElementKey = ExcalidrawElement | ElementIdKey;
 
 type SceneStateCallback = () => void;
 type SceneStateCallbackRemover = () => void;
 
-const isIdKey = (elementKey: ElementKey): elementKey is ElementIdKey => {
+const isIdKey = (elementKey: ElementKey): elementKey is string => {
   if (typeof elementKey === "string") {
     return true;
   }

src/scene/export.ts

@@ -67,6 +67,7 @@ export const exportToCanvas = async (
     renderSelection: false,
     renderGrid: false,
     isExporting: true,
+    editingImageElement: null,
   });
 
   return canvas;

src/scene/types.ts

@@ -11,6 +11,7 @@ export type RenderConfig = {
   zoom: AppState["zoom"];
   shouldCacheIgnoreZoom: AppState["shouldCacheIgnoreZoom"];
   theme: AppState["theme"];
+  editingImageElement: AppState["editingImageElement"];
   // collab-related state
   // ---------------------------------------------------------------------------
   remotePointerViewportCoords: { [id: string]: { x: number; y: number } };

src/types.ts

@@ -29,6 +29,8 @@ import { MaybeTransformHandleType } from "./element/transformHandles";
 import Library from "./data/library";
 import type { FileSystemHandle } from "./data/filesystem";
 import type { ALLOWED_IMAGE_MIME_TYPES, MIME_TYPES } from "./constants";
+import { EditingImageElement } from "./element/imageEditor";
+import Scene from "./scene/Scene";
 
 export type Point = Readonly<RoughPoint>;
 
@@ -77,6 +79,7 @@ export type AppState = {
   // (e.g. text element when typing into the input)
   editingElement: NonDeletedExcalidrawElement | null;
   editingLinearElement: LinearElementEditor | null;
+  editingImageElement: EditingImageElement | null;
   elementType: typeof SHAPES[number]["value"];
   elementLocked: boolean;
   exportBackground: boolean;
@@ -316,6 +319,8 @@ export type AppClassProperties = {
     }
   >;
   files: BinaryFiles;
+  scene: Scene;
+  addFiles: App["addFiles"];
 };
 
 export type PointerDownState = Readonly<{