There is a patch available for this at https://www.drupal.org/project/drupal/issues/2997613. However, it didn't work for me as I intended to use Bootstrap classes. Additionally, I need a flexible interface to select Bootstrap margin and padding classes.

After exploring a few comments and the core implementation, I followed the approach.

1. Alter the editor_media_dialog form and add a custom autocomplete field and a form submit handler.

/**
 * Implements hook_form_BASE_FORM_ID_alter().
 */
function dj_base_form_editor_media_dialog_alter(&$form, FormStateInterface $form_state, $form_id) {
  if (isset($form['align'])) {
    $cla_string = "";
    $editor_object = $form_state->getUserInput();
    if (array_key_exists('editor_object', $editor_object)) {
      if (isset($editor_object['editor_object']['classes']) &&
        !empty($editor_object['editor_object']['classes'])) {
        $classes = @$editor_object['editor_object']['classes'];
        // Prepare default values.
        if (!empty($classes)) {
          foreach ($classes as $key => $class) {
            $label = MediaHelper::getBsClassLabel($key);
            $cla_string .= "{$label} (" . $key . "),";
          }
          $cla_string = trim($cla_string, ',');
        }
      }
    }
    // Autocomplete field.
    $form['element_class'] = [
      '#type' => 'textfield',
      '#title' => t('Spacing'),
      '#autocomplete_route_name' => 'dj_base.bs_class_autocomplete',
      '#description' => t('Select padding and margin classes.'),
      '#default_value' => $cla_string,
    ];
    // Add submit handler.
    foreach (array_keys($form['actions']) as $action) {
      if ($action != 'preview' &&
        isset($form['actions'][$action]['#type']) &&
        $form['actions'][$action]['#type'] === 'submit') {
        $form['actions'][$action]['#submit'][] = 'dj_base_media_dialog_submit';
      }
    }
  }
}

 

2. Define submit handler and pass the class to the attribute array.

/**
 * Submit handler for editor_media_dialog form.
 *
 * @param array $form
 *   Form fields array.
 * @param \Drupal\Core\Form\FormStateInterface $form_state
 *   Form state object.
 */
function dj_base_media_dialog_submit(array &$form, FormStateInterface $form_state) {
  // Get selected classes.
  $classes = $form_state->getValue('element_class', []);
  if (!empty($classes)) {
    $attributes = $form_state->getValue('attributes', []);
    $classes = explode(',', $classes);

    $cls_string = '';
    foreach ($classes as $class) {
      $cls_string .= EntityAutocomplete::extractEntityIdFromAutocompleteInput($class) . " ";
    }
    $attributes['class'] = $cls_string;

    // Set embed class.
    $form_state->setValue('attributes', $attributes);
  }
}

Media helper class.

<?php

namespace Drupal\dj_base;

/**
 * Provides helper methods for dealing with media entities.
 */
class MediaHelper {

  /**
   * Return an array of Bootstrap spacing and margin classes.
   */
  public static function bsSpacingClasses() {
    return [
      "p-1" => "Padding 1",
      "p-2" => "Padding 2",
      "p-3" => "Padding 3",
      "p-4" => "Padding 4",
      "p-5" => "Padding 5",
      "pt-1" => "Padding Top 1",
      "pt-2" => "Padding Top 2",
      "pt-3" => "Padding Top 3",
      "pt-4" => "Padding Top 4",
      "pt-5" => "Padding Top 5",
      "pb-1" => "Padding Bottom 1",
      "pb-2" => "Padding Bottom 2",
      "pb-3" => "Padding Bottom 3",
      "pb-4" => "Padding Bottom 4",
      "pb-5" => "Padding Bottom 5",
      "m-1" => "Margin 1",
      "m-2" => "Margin 2",
      "m-3" => "Margin 3",
      "m-4" => "Margin 4",
      "m-5" => "Margin 5",
      "mt-1" => "Margin Top 1",
      "mt-2" => "Margin Top 2",
      "mt-3" => "Margin Top 3",
      "mt-4" => "Margin Top 4",
      "mt-5" => "Margin Top 5",
      "mb-1" => "Margin Bottom 1",
      "mb-2" => "Margin Bottom 2",
      "mb-3" => "Margin Bottom 3",
      "mb-4" => "Margin Bottom 4",
      "mb-5" => "Margin Bottom 5",
    ];
  }

  /**
   * Get bootstrap class label.
   *
   * @param string $class
   *   Class key.
   */
  public static function getBsClassLabel($class) {
    if ($class) {
      $classes = self::bsSpacingClasses();
      return $classes[$class] ?? "";
    }
    return "";
  }

}

Output:

Custom classes to drupal-media