diff --git a/packages/codemirror-graphql/src/utils/jsonParse.ts b/packages/codemirror-graphql/src/utils/jsonParse.ts
index b266a709789..92fded83cbf 100644
--- a/packages/codemirror-graphql/src/utils/jsonParse.ts
+++ b/packages/codemirror-graphql/src/utils/jsonParse.ts
@@ -155,8 +155,18 @@ function expect(str: string) {
   throw syntaxError(`Expected ${str} but found ${found}.`);
 }
 
+type SyntaxErrorPosition = { start: number; end: number };
+
+export class JSONSyntaxError extends Error {
+  readonly position: SyntaxErrorPosition;
+  constructor(message: string, position: SyntaxErrorPosition) {
+    super(message);
+    this.position = position;
+  }
+}
+
 function syntaxError(message: string) {
-  return { message, start, end };
+  return new JSONSyntaxError(message, { start, end });
 }
 
 function skip(k: string) {
diff --git a/packages/codemirror-graphql/src/variables/lint.ts b/packages/codemirror-graphql/src/variables/lint.ts
index ce6926c2687..11acd4ec725 100644
--- a/packages/codemirror-graphql/src/variables/lint.ts
+++ b/packages/codemirror-graphql/src/variables/lint.ts
@@ -18,6 +18,7 @@ import {
 } from 'graphql';
 
 import jsonParse, {
+  JSONSyntaxError,
   ParseArrayOutput,
   ParseObjectOutput,
   ParseValueOutput,
@@ -57,11 +58,11 @@ CodeMirror.registerHelper(
     let ast;
     try {
       ast = jsonParse(text);
-    } catch (syntaxError) {
-      if (syntaxError.stack) {
-        throw syntaxError;
+    } catch (error) {
+      if (error instanceof JSONSyntaxError) {
+        return [lintError(editor, error.position, error.message)];
       }
-      return [lintError(editor, syntaxError, syntaxError.message)];
+      throw error;
     }
 
     // If there are not yet known variables, do nothing.
diff --git a/packages/graphiql/src/components/ToolbarButton.tsx b/packages/graphiql/src/components/ToolbarButton.tsx
index b3043503996..45d32da5671 100644
--- a/packages/graphiql/src/components/ToolbarButton.tsx
+++ b/packages/graphiql/src/components/ToolbarButton.tsx
@@ -49,7 +49,11 @@ export class ToolbarButton extends React.Component<
       this.props.onClick();
       this.setState({ error: null });
     } catch (error) {
-      this.setState({ error });
+      if (error instanceof Error) {
+        this.setState({ error });
+        return;
+      }
+      throw error;
     }
   };
 }
diff --git a/packages/graphql-language-service-cli/src/cli.ts b/packages/graphql-language-service-cli/src/cli.ts
index 61c3ec8e31e..f357f34bda9 100644
--- a/packages/graphql-language-service-cli/src/cli.ts
+++ b/packages/graphql-language-service-cli/src/cli.ts
@@ -130,7 +130,7 @@ switch (command) {
       startServer(options);
     } catch (error) {
       const logger = new Logger();
-      logger.error(error);
+      logger.error(String(error));
     }
     break;
   default: {
diff --git a/packages/graphql-language-service-cli/src/client.ts b/packages/graphql-language-service-cli/src/client.ts
index 5861ffd0a03..1b161279695 100644
--- a/packages/graphql-language-service-cli/src/client.ts
+++ b/packages/graphql-language-service-cli/src/client.ts
@@ -79,6 +79,14 @@ interface AutocompleteResultsMap {
   [i: number]: CompletionItem;
 }
 
+function formatUnknownError(error: unknown) {
+  let message: string | undefined;
+  if (error instanceof Error) {
+    message = error.stack;
+  }
+  return message ?? String(error);
+}
+
 function _getAutocompleteSuggestions(
   queryText: string,
   point: Position,
@@ -104,7 +112,7 @@ function _getAutocompleteSuggestions(
     process.stdout.write(JSON.stringify(resultObject, null, 2));
     return GRAPHQL_SUCCESS_CODE;
   } catch (error) {
-    process.stderr.write((error?.stack ?? String(error)) + '\n');
+    process.stderr.write(formatUnknownError(error) + '\n');
     return GRAPHQL_FAILURE_CODE;
   }
 }
@@ -133,7 +141,7 @@ function _getDiagnostics(
     process.stdout.write(JSON.stringify(resultObject, null, 2));
     return GRAPHQL_SUCCESS_CODE;
   } catch (error) {
-    process.stderr.write((error?.stack ?? String(error)) + '\n');
+    process.stderr.write(formatUnknownError(error) + '\n');
     return GRAPHQL_FAILURE_CODE;
   }
 }
@@ -147,7 +155,7 @@ function _getOutline(queryText: string): EXIT_CODE {
       throw Error('Error parsing or no outline tree found');
     }
   } catch (error) {
-    process.stderr.write((error?.stack ?? String(error)) + '\n');
+    process.stderr.write(formatUnknownError(error) + '\n');
     return GRAPHQL_FAILURE_CODE;
   }
   return GRAPHQL_SUCCESS_CODE;
@@ -161,7 +169,7 @@ function ensureText(queryText: string, filePath: string): string {
     try {
       text = fs.readFileSync(filePath, 'utf8');
     } catch (error) {
-      throw new Error(error);
+      throw new Error(String(error));
     }
   }
   return text;
diff --git a/packages/graphql-language-service-server/src/GraphQLLanguageService.ts b/packages/graphql-language-service-server/src/GraphQLLanguageService.ts
index 709f9ae8446..e8048a2fc99 100644
--- a/packages/graphql-language-service-server/src/GraphQLLanguageService.ts
+++ b/packages/graphql-language-service-server/src/GraphQLLanguageService.ts
@@ -14,6 +14,7 @@ import {
   TypeDefinitionNode,
   NamedTypeNode,
   ValidationRule,
+  GraphQLError,
 } from 'graphql';
 
 import {
@@ -153,15 +154,22 @@ export class GraphQLLanguageService {
         });
       }
     } catch (error) {
-      const range = getRange(error.locations[0], document);
-      return [
-        {
-          severity: DIAGNOSTIC_SEVERITY.Error,
-          message: error.message,
-          source: 'GraphQL: Syntax',
-          range,
-        },
-      ];
+      if (error instanceof GraphQLError) {
+        const range = getRange(
+          error.locations?.[0] ?? { column: 0, line: 0 },
+          document,
+        );
+        return [
+          {
+            severity: DIAGNOSTIC_SEVERITY.Error,
+            message: error.message,
+            source: 'GraphQL: Syntax',
+            range,
+          },
+        ];
+      }
+
+      throw error;
     }
 
     // If there's a matching config, proceed to prepare to run validation
diff --git a/packages/graphql-language-service-server/src/MessageProcessor.ts b/packages/graphql-language-service-server/src/MessageProcessor.ts
index dd0a4ee9128..7ecca494cf2 100644
--- a/packages/graphql-language-service-server/src/MessageProcessor.ts
+++ b/packages/graphql-language-service-server/src/MessageProcessor.ts
@@ -256,7 +256,7 @@ export class MessageProcessor {
         }
       }
     } catch (err) {
-      this._logger.warn(err);
+      this._logger.warn(String(err));
     }
 
     // Here, we set the workspace settings in memory,
@@ -834,7 +834,7 @@ export class MessageProcessor {
         await this._updateObjectTypeDefinition(uri, contents);
       }
     } catch (err) {
-      this._logger.error(err);
+      this._logger.error(String(err));
     }
   }
   async _cacheSchemaFile(
@@ -1002,7 +1002,7 @@ export class MessageProcessor {
         }
       }
     } catch (err) {
-      this._logger.error(err);
+      this._logger.error(String(err));
     }
   }
   /**
@@ -1043,7 +1043,7 @@ export class MessageProcessor {
       this._logger.error(
         `invalid/unknown file in graphql config documents entry:\n '${project.documents}'`,
       );
-      this._logger.error(err);
+      this._logger.error(String(err));
     }
   }
   /**
diff --git a/packages/graphql-language-service-server/src/findGraphQLTags.ts b/packages/graphql-language-service-server/src/findGraphQLTags.ts
index d0b80cb4950..370d96e6605 100644
--- a/packages/graphql-language-service-server/src/findGraphQLTags.ts
+++ b/packages/graphql-language-service-server/src/findGraphQLTags.ts
@@ -95,7 +95,7 @@ export function findGraphQLTags(
     logger.error(
       `Could not parse the ${type} file at ${uri} to extract the graphql tags:`,
     );
-    logger.error(error);
+    logger.error(String(error));
     return [];
   }
   const ast = parsedAST!;
diff --git a/packages/graphql-language-service-server/src/startServer.ts b/packages/graphql-language-service-server/src/startServer.ts
index 58ef94b3e9f..9bc8d8d2d47 100644
--- a/packages/graphql-language-service-server/src/startServer.ts
+++ b/packages/graphql-language-service-server/src/startServer.ts
@@ -211,7 +211,7 @@ export default async function startServer(
       serverWithHandlers.listen();
     } catch (err) {
       logger.error('There was a Graphql LSP handler exception:');
-      logger.error(err);
+      logger.error(String(err));
     }
   }
 }
@@ -236,7 +236,7 @@ async function initializeHandlers({
     return connection;
   } catch (err) {
     logger.error('There was an error initializing the server connection');
-    logger.error(err);
+    logger.error(String(err));
     process.exit(1);
   }
 }
diff --git a/packages/graphql-language-service/src/interface/getDiagnostics.ts b/packages/graphql-language-service/src/interface/getDiagnostics.ts
index 52f84bc88fc..0975fb795c5 100644
--- a/packages/graphql-language-service/src/interface/getDiagnostics.ts
+++ b/packages/graphql-language-service/src/interface/getDiagnostics.ts
@@ -81,15 +81,22 @@ export function getDiagnostics(
   try {
     ast = parse(query);
   } catch (error) {
-    const range = getRange(error.locations[0], query);
-    return [
-      {
-        severity: DIAGNOSTIC_SEVERITY.Error as DiagnosticSeverity,
-        message: error.message,
-        source: 'GraphQL: Syntax',
-        range,
-      },
-    ];
+    if (error instanceof GraphQLError) {
+      const range = getRange(
+        error.locations?.[0] ?? { line: 0, column: 0 },
+        query,
+      );
+
+      return [
+        {
+          severity: DIAGNOSTIC_SEVERITY.Error as DiagnosticSeverity,
+          message: error.message,
+          source: 'GraphQL: Syntax',
+          range,
+        },
+      ];
+    }
+    throw error;
   }
 
   return validateQuery(ast, schema, customRules, isRelayCompatMode);
diff --git a/packages/graphql-language-service/src/parser/Rules.ts b/packages/graphql-language-service/src/parser/Rules.ts
index 016eba2f389..f253a691863 100644
--- a/packages/graphql-language-service/src/parser/Rules.ts
+++ b/packages/graphql-language-service/src/parser/Rules.ts
@@ -196,7 +196,7 @@ export const ParseRules: { [name: string]: ParseRule } = {
   StringValue: [
     {
       style: 'string',
-      match: token => token.kind === 'String',
+      match: (token: Token) => token.kind === 'String',
       update(state: State, token: Token) {
         if (token.value.startsWith('"""')) {
           state.inBlockstring = !token.value.slice(3).endsWith('"""');