This issue is a Microsoft bug that occurs within the HoloLens OS which Unity cannot fix. Please file a bug with Microsoft for this problem.

The best we can offer is a work around which activates the Windows.Media.SpeechRecognition.SpeechRecognizer COM object on a separate thread allowing Unity's main thread to continue (Update and Render). Once COM activation finally completes, KeywordRecognizer can be instantiated without any delay, since it'll merely adds RefCount to the existing COM object.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.Windows.Speech;

public class testscript : MonoBehaviour {
bool initialized = false;
bool directInitComplete = false;

public Text text;
// Use this for initialization
void Start () {
InitializeWindowsSpeechDirectly();
}

// Update is called once per frame
void Update () {
if (!initialized && directInitComplete)
{
Debug.Log("Creating KeywordRecognizer!");

float timeStart = Time.realtimeSinceStartup;
var r = new KeywordRecognizer(new string[] { "test-keyword" }, ConfidenceLevel.Low);
float totalTime = Time.realtimeSinceStartup - timeStart;
Debug.Log("new KeywordRecognizer took " + totalTime + "s");

if (text != null) text.text = "new KeywordRecognizer took " + totalTime + "s";

r.OnPhraseRecognized += (args) =>
{
Debug.Log("Phrase recognized!");
};

Debug.Log("Starting KeywordRecognizer!");
r.Start();

initialized = true;

Debug.Log("Finished initialization!");
}
}

#if ENABLE_WINMD_SUPPORT
Windows.Media.SpeechRecognition.SpeechRecognizer _recognizer;
#endif

private void InitializeWindowsSpeechDirectly()
{
#if ENABLE_WINMD_SUPPORT

Windows.System.Threading.ThreadPool.RunAsync((workItem) =>
{
// WinRT APIs will throw exceptions
try
{
var recogonizer = new Windows.Media.SpeechRecognition.SpeechRecognizer();

_recognizer = recogonizer;
directInitComplete = true;
}
catch { };
});
#else
directInitComplete = true;
#endif

}
}