Issue #2472 - Part 6: Fire cancel event and close modal <dialog> on [ESC]

This implementation is different than Mozilla, not indirectly queueing cancel
tasks to be arbitrarily ordered, but rather synchronously close in reverse
top layer order.

dom/base/nsDocument.cpp

@@ -159,6 +159,7 @@
 #include "nsEscape.h"
 #include "nsObjectLoadingContent.h"
 #include "nsHtml5TreeOpExecutor.h"
+#include "mozilla/dom/HTMLDialogElement.h"
 #include "mozilla/dom/HTMLLinkElement.h"
 #include "mozilla/dom/HTMLMediaElement.h"
 #include "mozilla/dom/HTMLIFrameElement.h"
@@ -10138,6 +10139,18 @@ nsDocument::MozCancelFullScreen()
   return NS_OK;
 }
 
+void nsDocument::TryCancelDialog() {
+  // Check if the document is blocked by modal dialog
+  for (const nsWeakPtr& weakPtr : Reversed(mTopLayer)) {
+    nsCOMPtr<Element> element(do_QueryReferent(weakPtr));
+    if (HTMLDialogElement* dialog =
+            HTMLDialogElement::FromContentOrNull(element)) {
+      dialog->CancelDialog();
+      break;
+    }
+  }
+}
+
 void
 nsIDocument::ExitFullscreen()
 {

dom/base/nsDocument.h

@@ -918,6 +918,9 @@ public:
   // flag.
   bool SetFullscreenElement(Element* aElement);
 
+  // Cancel the dialog element if the document is blocked by the dialog
+  void TryCancelDialog();
+
   // Returns the top element from the full-screen stack.
   Element* GetTopLayerTop() override;
   // Return the fullscreen element in the top layer

dom/base/nsIDocument.h

@@ -1349,6 +1349,11 @@ public:
   virtual bool TopLayerPush(Element* aElement) = 0;
   virtual Element* TopLayerPop(FunctionRef<bool(Element*)> aPredicateFunc) = 0;
 
+  /**
+   * Cancel the dialog element if the document is blocked by the dialog.
+   */
+  virtual void TryCancelDialog() = 0;
+  
   /**
    * Synchronously cleans up the fullscreen state on the given document.
    *

dom/html/HTMLDialogElement.cpp

@@ -111,6 +111,27 @@ HTMLDialogElement::ShowModal(ErrorResult& aError)
   aError.SuppressException();
 }
 
+void HTMLDialogElement::CancelDialog() {
+  // 1) Let close be the result of firing an event named cancel at dialog, with
+  // the cancelable attribute initialized to true.
+  bool defaultAction = true;
+  nsContentUtils::DispatchTrustedEvent(
+      OwnerDoc(),
+      static_cast<nsIContent*>(this),
+      NS_LITERAL_STRING("cancel"),
+      false, /* can bubble */
+      true, /* can cancel */
+      &defaultAction);
+
+  // 2) If close is true and dialog has an open attribute, then close the dialog
+  // with no return value.
+  if (defaultAction) {
+    Optional<nsAString> retValue;
+    Close(retValue);
+  }
+}
+
+
 JSObject*
 HTMLDialogElement::WrapNode(JSContext* aCx, JS::Handle<JSObject*> aGivenProto)
 {

dom/html/HTMLDialogElement.h

@@ -49,6 +49,7 @@ public:
   void ShowModal(ErrorResult& aError);
   
   bool IsInTopLayer() const;
+  void CancelDialog();
 
   nsString mReturnValue;
 

layout/base/nsPresShell.cpp

@@ -8059,6 +8059,12 @@ PresShell::HandleEventInternal(WidgetEvent* aEvent,
               aEvent->mFlags.mDefaultPreventedByChrome) {
             mIsLastChromeOnlyEscapeKeyConsumed = true;
           }
+          if (aEvent->mMessage == eKeyDown &&
+              !aEvent->mFlags.mDefaultPrevented) {
+            if (nsIDocument* doc = GetDocument()) {
+              doc->TryCancelDialog();
+            }
+          }
         }
       }
       break;